From 335f8c5100d6922dc95d87bde2c590c0ded543b1 Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Thu, 10 Jan 2019 10:17:50 -0800 Subject: [PATCH] Update to PMIx 3.1.2 Update the OPAL glue configure code to correctly link the opal/pmix3 component to the hwloc used by OMPI instead of defaulting to the system-level hwloc. Required a corresponding update to the PMIx hwloc configure code so we treat hwloc the same way we handle libevent in embedded scenarios. Roll to PMIx v3.1.2 for plugging of memory leaks and addition of faster PMIx_Get response Signed-off-by: Ralph Castain --- .gitignore | 2 + opal/mca/pmix/pmix3x/configure.m4 | 6 +- opal/mca/pmix/pmix3x/pmix/INSTALL | 24 +- opal/mca/pmix/pmix3x/pmix/LICENSE | 6 +- opal/mca/pmix/pmix3x/pmix/Makefile.am | 4 - opal/mca/pmix/pmix3x/pmix/NEWS | 99 +- opal/mca/pmix/pmix3x/pmix/README | 27 +- opal/mca/pmix/pmix3x/pmix/VERSION | 15 +- opal/mca/pmix/pmix3x/pmix/config/pmix.m4 | 51 +- .../pmix3x/pmix/config/pmix_check_lock.m4 | 2 + .../pmix/config/pmix_check_os_flavors.m4 | 9 +- .../pmix3x/pmix/config/pmix_config_asm.m4 | 593 +++-- opal/mca/pmix/pmix3x/pmix/config/pmix_mca.m4 | 4 +- .../pmix/pmix3x/pmix/config/pmix_setup_cc.m4 | 13 +- .../pmix/pmix3x/pmix/config/pmix_setup_cli.m4 | 4 +- .../pmix3x/pmix/config/pmix_setup_hwloc.m4 | 78 +- .../pmix3x/pmix/config/pmix_setup_libevent.m4 | 9 +- .../pmix/pmix3x/pmix/config/pmix_summary.m4 | 79 + opal/mca/pmix/pmix3x/pmix/configure.ac | 6 +- opal/mca/pmix/pmix3x/pmix/contrib/Makefile.am | 2 - .../pmix/contrib/cron-run-all-md2nroff.pl | 187 -- opal/mca/pmix/pmix3x/pmix/contrib/md2nroff.pl | 164 -- opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec | 2 +- .../mca/pmix/pmix3x/pmix/examples/Makefile.am | 29 +- opal/mca/pmix/pmix3x/pmix/examples/alloc.c | 133 +- opal/mca/pmix/pmix3x/pmix/examples/client.c | 216 +- opal/mca/pmix/pmix3x/pmix/examples/client2.c | 28 +- opal/mca/pmix/pmix3x/pmix/examples/dmodex.c | 24 +- opal/mca/pmix/pmix3x/pmix/examples/dynamic.c | 4 +- opal/mca/pmix/pmix3x/pmix/examples/examples.h | 124 + opal/mca/pmix/pmix3x/pmix/examples/fault.c | 97 +- opal/mca/pmix/pmix3x/pmix/examples/jctrl.c | 71 +- opal/mca/pmix/pmix3x/pmix/examples/tool.c | 179 +- opal/mca/pmix/pmix3x/pmix/include/Makefile.am | 5 +- opal/mca/pmix/pmix3x/pmix/include/pmix.h | 12 +- .../pmix/pmix3x/pmix/include/pmix_common.h.in | 961 +++++--- .../pmix/pmix3x/pmix/include/pmix_extend.h | 155 ++ .../pmix/pmix3x/pmix/include/pmix_server.h | 8 +- .../pmix3x/pmix/include/pmix_version.h.in | 4 +- opal/mca/pmix/pmix3x/pmix/man/Makefile.am | 60 - opal/mca/pmix/pmix3x/pmix/man/README | 186 -- .../pmix/pmix3x/pmix/man/man3/pmix_abort.3 | 62 - .../pmix/pmix3x/pmix/man/man3/pmix_commit.3 | 35 - .../pmix/pmix3x/pmix/man/man3/pmix_finalize.3 | 45 - .../mca/pmix/pmix3x/pmix/man/man3/pmix_init.3 | 52 - .../pmix3x/pmix/man/man3/pmix_initialized.3 | 30 - opal/mca/pmix/pmix3x/pmix/man/man3/pmix_put.3 | 60 - opal/mca/pmix/pmix3x/pmix/man/man7/pmix.7 | 35 - .../pmix3x/pmix/man/man7/pmix_constants.7 | 79 - .../pmix/src/atomics/sys/Makefile.include | 5 +- .../pmix/src/atomics/sys/architecture.h | 3 +- .../pmix3x/pmix/src/atomics/sys/arm/atomic.h | 20 +- .../pmix/src/atomics/sys/arm64/atomic.h | 30 +- .../pmix/pmix3x/pmix/src/atomics/sys/atomic.h | 146 +- .../pmix3x/pmix/src/atomics/sys/atomic_impl.h | 94 +- .../pmix3x/pmix/src/atomics/sys/atomic_stdc.h | 262 +++ .../pmix/pmix3x/pmix/src/atomics/sys/cma.h | 10 +- .../pmix/src/atomics/sys/gcc_builtin/atomic.h | 46 +- .../pmix3x/pmix/src/atomics/sys/ia32/atomic.h | 20 +- .../pmix/src/atomics/sys/powerpc/atomic.h | 32 +- .../pmix/src/atomics/sys/sparcv9/atomic.h | 26 +- .../src/atomics/sys/sync_builtin/atomic.h | 30 +- .../pmix/pmix3x/pmix/src/atomics/sys/timer.h | 2 +- .../pmix/src/atomics/sys/x86_64/atomic.h | 38 +- .../pmix/src/atomics/sys/x86_64/timer.h | 2 +- .../pmix/pmix3x/pmix/src/class/pmix_hotel.h | 16 +- .../pmix/pmix3x/pmix/src/class/pmix_list.h | 4 +- .../pmix/pmix3x/pmix/src/class/pmix_object.h | 2 +- opal/mca/pmix/pmix3x/pmix/src/client/pmi1.c | 20 +- opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c | 30 +- .../pmix/pmix3x/pmix/src/client/pmix_client.c | 58 +- .../pmix/src/client/pmix_client_fence.c | 4 +- .../pmix3x/pmix/src/client/pmix_client_get.c | 121 +- .../pmix3x/pmix/src/client/pmix_client_pub.c | 2 +- .../pmix/src/client/pmix_client_spawn.c | 113 +- .../pmix3x/pmix/src/common/pmix_control.c | 4 + .../pmix/pmix3x/pmix/src/common/pmix_data.c | 14 +- .../pmix/pmix3x/pmix/src/common/pmix_iof.h | 2 +- .../pmix/pmix3x/pmix/src/common/pmix_query.c | 169 +- .../pmix3x/pmix/src/common/pmix_strings.c | 13 +- .../pmix/pmix3x/pmix/src/event/pmix_event.h | 135 +- .../pmix/src/event/pmix_event_notification.c | 316 ++- .../pmix/src/event/pmix_event_registration.c | 66 +- .../pmix3x/pmix/src/hwloc/hwloc-internal.h | 4 +- opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c | 4 +- .../pmix3x/pmix/src/include/Makefile.include | 5 +- .../pmix/src/include/pmix_config_bottom.h | 3 +- .../pmix3x/pmix/src/include/pmix_globals.c | 105 +- .../pmix3x/pmix/src/include/pmix_globals.h | 33 +- .../pmix3x/pmix/src/include/pmix_stdatomic.h | 67 + .../pmix3x/pmix/src/include/pmix_stdint.h | 5 +- .../base/pmix_mca_base_component_repository.c | 8 +- .../pmix3x/pmix/src/mca/bfrops/base/base.h | 24 +- .../src/mca/bfrops/base/bfrop_base_copy.c | 83 +- .../pmix/src/mca/bfrops/base/bfrop_base_fns.c | 23 +- .../src/mca/bfrops/base/bfrop_base_pack.c | 69 +- .../src/mca/bfrops/base/bfrop_base_print.c | 46 - .../src/mca/bfrops/base/bfrop_base_unpack.c | 99 +- .../pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h | 2 +- .../pmix3x/pmix/src/mca/bfrops/v12/copy.c | 33 +- .../pmix3x/pmix/src/mca/bfrops/v12/internal.h | 53 +- .../pmix3x/pmix/src/mca/bfrops/v12/pack.c | 7 +- .../pmix3x/pmix/src/mca/bfrops/v12/unpack.c | 16 +- .../pmix3x/pmix/src/mca/bfrops/v20/copy.c | 988 ++++---- .../pmix3x/pmix/src/mca/bfrops/v20/internal.h | 53 +- .../pmix3x/pmix/src/mca/bfrops/v20/pack.c | 17 +- .../pmix3x/pmix/src/mca/bfrops/v20/print.c | 10 +- .../pmix3x/pmix/src/mca/bfrops/v20/unpack.c | 21 +- .../pmix/src/mca/bfrops/v21/bfrop_pmix21.c | 244 +- .../pmix/src/mca/bfrops/v3/bfrop_pmix3.c | 245 +- .../pmix3x/pmix/src/mca/common/Makefile.am | 25 + .../pmix/src/mca/common/dstore/Makefile.am | 59 + .../pmix/src/mca/common/dstore/configure.m4 | 17 + .../dstore/dstore_base.c} | 2059 +++++++---------- .../pmix/src/mca/common/dstore/dstore_base.h | 139 ++ .../src/mca/common/dstore/dstore_common.h | 79 + .../pmix/src/mca/common/dstore/dstore_file.h | 161 ++ .../src/mca/common/dstore/dstore_segment.c | 302 +++ .../src/mca/common/dstore/dstore_segment.h | 70 + .../pmix/pmix3x/pmix/src/mca/gds/base/base.h | 13 + .../pmix/src/mca/gds/base/gds_base_fns.c | 99 + .../pmix3x/pmix/src/mca/gds/ds12/Makefile.am | 24 +- .../pmix/src/mca/gds/ds12/gds_ds12_base.c | 170 ++ .../pmix/src/mca/gds/ds12/gds_ds12_base.h | 21 + ...store_component.c => gds_ds12_component.c} | 2 +- .../pmix/src/mca/gds/ds12/gds_ds12_file.c | 176 ++ .../pmix/src/mca/gds/ds12/gds_ds12_file.h | 20 + .../pmix/src/mca/gds/ds12/gds_ds12_lock.c | 25 + .../pmix/src/mca/gds/ds12/gds_ds12_lock.h | 29 + .../src/mca/gds/ds12/gds_ds12_lock_fcntl.c | 233 ++ .../src/mca/gds/ds12/gds_ds12_lock_pthread.c | 274 +++ .../pmix/src/mca/gds/ds12/gds_ds20_file.c | 166 ++ .../pmix3x/pmix/src/mca/gds/ds12/gds_dstore.h | 129 -- .../pmix3x/pmix/src/mca/gds/ds21/Makefile.am | 62 + .../pmix/src/mca/gds/ds21/gds_ds21_base.c | 176 ++ .../pmix/src/mca/gds/ds21/gds_ds21_base.h | 21 + .../src/mca/gds/ds21/gds_ds21_component.c | 93 + .../pmix/src/mca/gds/ds21/gds_ds21_file.c | 188 ++ .../pmix/src/mca/gds/ds21/gds_ds21_file.h | 19 + .../pmix/src/mca/gds/ds21/gds_ds21_lock.c | 25 + .../pmix/src/mca/gds/ds21/gds_ds21_lock.h | 30 + .../src/mca/gds/ds21/gds_ds21_lock_pthread.c | 438 ++++ opal/mca/pmix/pmix3x/pmix/src/mca/gds/gds.h | 37 +- .../pmix3x/pmix/src/mca/gds/hash/gds_hash.c | 41 +- .../pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c | 19 +- .../src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c | 19 +- .../src/mca/pif/linux_ipv6/pif_linux_ipv6.c | 19 +- opal/mca/pmix/pmix3x/pmix/src/mca/pif/pif.h | 18 +- .../pmix/src/mca/pif/posix_ipv4/pif_posix.c | 18 +- .../mca/pif/solaris_ipv6/pif_solaris_ipv6.c | 8 +- .../src/mca/pinstalldirs/config/Makefile.am | 4 +- .../pmix/src/mca/plog/stdfd/plog_stdfd.c | 2 +- .../pmix/pmix3x/pmix/src/mca/pnet/base/base.h | 2 +- .../pmix/src/mca/pnet/base/pnet_base_fns.c | 76 +- .../pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c | 49 +- .../pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h | 4 +- .../src/mca/pnet/opa/pnet_opa_component.c | 21 +- opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h | 12 +- .../pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c | 77 +- .../pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h | 4 +- .../src/mca/pnet/tcp/pnet_tcp_component.c | 21 +- .../pmix3x/pmix/src/mca/pnet/test/pnet_test.c | 42 +- .../pmix/src/mca/preg/base/preg_base_frame.c | 2 + .../pmix/src/mca/preg/native/preg_native.c | 24 +- .../pmix3x/pmix/src/mca/preg/preg_types.h | 2 + .../pmix/src/mca/psensor/file/psensor_file.c | 2 +- .../mca/psensor/heartbeat/psensor_heartbeat.c | 2 +- .../src/mca/pshmem/base/pshmem_base_frame.c | 5 +- .../pmix/src/mca/pshmem/mmap/pshmem_mmap.c | 4 +- .../pmix/src/mca/ptl/base/ptl_base_frame.c | 4 + .../pmix/src/mca/ptl/base/ptl_base_sendrecv.c | 169 +- opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h | 20 +- .../pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h | 88 +- .../pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c | 610 +++-- .../pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h | 4 + .../pmix/src/mca/ptl/tcp/ptl_tcp_component.c | 555 +++-- .../pmix3x/pmix/src/mca/ptl/usock/ptl_usock.c | 4 +- .../src/mca/ptl/usock/ptl_usock_component.c | 13 +- .../pmix3x/pmix/src/runtime/pmix_finalize.c | 10 +- .../pmix/pmix3x/pmix/src/runtime/pmix_init.c | 41 +- .../pmix3x/pmix/src/runtime/pmix_params.c | 28 +- .../pmix/pmix3x/pmix/src/server/pmix_server.c | 863 ++++--- .../pmix3x/pmix/src/server/pmix_server_get.c | 156 +- .../pmix3x/pmix/src/server/pmix_server_ops.c | 866 ++++--- .../pmix3x/pmix/src/server/pmix_server_ops.h | 39 +- .../pmix/pmix3x/pmix/src/threads/mutex_unix.h | 8 +- .../pmix3x/pmix/src/threads/thread_usage.h | 19 +- .../pmix/pmix3x/pmix/src/threads/wait_sync.h | 2 +- .../mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c | 147 +- .../pmix3x/pmix/src/tools/pevent/Makefile.am | 3 +- .../pmix3x/pmix/src/tools/pevent/pevent.c | 6 +- .../pmix3x/pmix/src/tools/plookup/Makefile.am | 5 +- .../pmix3x/pmix/src/tools/plookup/plookup.c | 4 +- .../pmix/src/tools/pmix_info/Makefile.am | 4 +- .../pmix3x/pmix/src/tools/pps/Makefile.am | 5 +- opal/mca/pmix/pmix3x/pmix/src/util/argv.c | 6 +- opal/mca/pmix/pmix3x/pmix/src/util/basename.c | 8 +- opal/mca/pmix/pmix3x/pmix/src/util/error.c | 26 +- opal/mca/pmix/pmix3x/pmix/src/util/hash.c | 4 +- .../pmix/pmix3x/pmix/src/util/keyval_parse.c | 6 +- opal/mca/pmix/pmix3x/pmix/src/util/net.c | 16 - opal/mca/pmix/pmix3x/pmix/src/util/os_path.c | 4 +- opal/mca/pmix/pmix3x/pmix/src/util/os_path.h | 9 +- opal/mca/pmix/pmix3x/pmix/src/util/output.c | 4 +- opal/mca/pmix/pmix3x/pmix/src/util/pif.c | 26 +- .../mca/pmix/pmix3x/pmix/src/util/show_help.c | 16 +- .../mca/pmix/pmix3x/pmix/src/util/show_help.h | 15 +- opal/mca/pmix/pmix3x/pmix/test/Makefile.am | 8 +- opal/mca/pmix/pmix3x/pmix/test/cli_stages.c | 23 +- opal/mca/pmix/pmix3x/pmix/test/cli_stages.h | 11 +- opal/mca/pmix/pmix3x/pmix/test/pmix_client.c | 7 +- opal/mca/pmix/pmix3x/pmix/test/pmix_test.c | 86 +- .../pmix/pmix3x/pmix/test/server_callbacks.c | 78 +- .../pmix/pmix3x/pmix/test/simple/simpclient.c | 1 + .../pmix/pmix3x/pmix/test/simple/simpdie.c | 8 +- .../pmix/pmix3x/pmix/test/simple/simpdmodex.c | 4 +- .../pmix/pmix3x/pmix/test/simple/simptest.c | 234 +- .../pmix/pmix3x/pmix/test/simple/simptool.c | 3 +- opal/mca/pmix/pmix3x/pmix/test/test_common.c | 13 +- opal/mca/pmix/pmix3x/pmix/test/test_common.h | 14 +- opal/mca/pmix/pmix3x/pmix/test/test_server.c | 978 ++++++++ opal/mca/pmix/pmix3x/pmix/test/test_server.h | 80 + opal/mca/pmix/pmix3x/pmix/test/utils.c | 179 +- opal/mca/pmix/pmix3x/pmix/test/utils.h | 5 +- 224 files changed, 12901 insertions(+), 6724 deletions(-) create mode 100644 opal/mca/pmix/pmix3x/pmix/config/pmix_summary.m4 delete mode 100755 opal/mca/pmix/pmix3x/pmix/contrib/cron-run-all-md2nroff.pl delete mode 100755 opal/mca/pmix/pmix3x/pmix/contrib/md2nroff.pl create mode 100644 opal/mca/pmix/pmix3x/pmix/examples/examples.h create mode 100644 opal/mca/pmix/pmix3x/pmix/include/pmix_extend.h delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/Makefile.am delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/README delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/man3/pmix_abort.3 delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/man3/pmix_commit.3 delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/man3/pmix_finalize.3 delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/man3/pmix_init.3 delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/man3/pmix_initialized.3 delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/man3/pmix_put.3 delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/man7/pmix.7 delete mode 100644 opal/mca/pmix/pmix3x/pmix/man/man7/pmix_constants.7 create mode 100644 opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic_stdc.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/include/pmix_stdatomic.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/common/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/configure.m4 rename opal/mca/pmix/pmix3x/pmix/src/mca/{gds/ds12/gds_dstore.c => common/dstore/dstore_base.c} (55%) create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_base.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_common.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_file.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_segment.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_segment.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_base.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_base.h rename opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/{gds_dstore_component.c => gds_ds12_component.c} (99%) create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_file.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_file.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock_fcntl.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock_pthread.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds20_file.c delete mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/Makefile.am create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_base.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_base.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_component.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_file.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_file.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock.c create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock.h create mode 100644 opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c create mode 100644 opal/mca/pmix/pmix3x/pmix/test/test_server.c create mode 100644 opal/mca/pmix/pmix3x/pmix/test/test_server.h diff --git a/.gitignore b/.gitignore index 07ae3be802..d40aac1e60 100644 --- a/.gitignore +++ b/.gitignore @@ -365,6 +365,8 @@ opal/mca/hwloc/base/static-components.h.new.struct opal/mca/installdirs/config/install_dirs.h +!opal/mca/pmix/pmix*/pmix/AUTHORS +!opal/mca/pmix/pmix*/pmix/contrib/perf_tools/Makefile opal/mca/pmix/pmix*/pmix/include/pmix/autogen/config.h opal/mca/pmix/pmix*/pmix/include/pmix/autogen/config.h.in opal/mca/pmix/pmix*/pmix/src/include/private/autogen/config.h.in diff --git a/opal/mca/pmix/pmix3x/configure.m4 b/opal/mca/pmix/pmix3x/configure.m4 index 6188960eb3..10cd639e09 100644 --- a/opal/mca/pmix/pmix3x/configure.m4 +++ b/opal/mca/pmix/pmix3x/configure.m4 @@ -13,7 +13,7 @@ # Copyright (c) 2011-2013 Los Alamos National Security, LLC. # All rights reserved. # Copyright (c) 2010-2017 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2015-2016 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -53,14 +53,14 @@ AC_DEFUN([MCA_opal_pmix_pmix3x_CONFIG],[ opal_pmix_pmix3x_timing_flag=--disable-pmix-timing fi - opal_pmix_pmix3x_args="$opal_pmix_pmix3x_timing_flag --without-tests-examples --disable-pmix-binaries --disable-pmix-backward-compatibility --disable-visibility --enable-embedded-libevent --with-libevent-header=\\\"opal/mca/event/$opal_event_base_include\\\"" + opal_pmix_pmix3x_args="$opal_pmix_pmix3x_timing_flag --without-tests-examples --disable-pmix-binaries --disable-pmix-backward-compatibility --disable-visibility --enable-embedded-libevent --with-libevent-header=\\\"opal/mca/event/$opal_event_base_include\\\" --enable-embedded-hwloc --with-hwloc-header=\\\"$opal_hwloc_base_include\\\"" AS_IF([test "$enable_debug" = "yes"], [opal_pmix_pmix3x_args="--enable-debug $opal_pmix_pmix3x_args" CFLAGS="$OPAL_CFLAGS_BEFORE_PICKY $OPAL_VISIBILITY_CFLAGS -g"], [opal_pmix_pmix3x_args="--disable-debug $opal_pmix_pmix3x_args" CFLAGS="$OPAL_CFLAGS_BEFORE_PICKY $OPAL_VISIBILITY_CFLAGS"]) AC_MSG_CHECKING([if want to install standalone libpmix]) - AS_IF([test "$enable_install_libpmix" == "yes"], + AS_IF([test "$enable_install_libpmix" = "yes"], [AC_MSG_RESULT([yes])], [AC_MSG_RESULT([no]) opal_pmix_pmix3x_args="--with-pmix-symbol-rename=OPAL_MCA_PMIX3X_ --enable-embedded-mode $opal_pmix_pmix3x_args"]) diff --git a/opal/mca/pmix/pmix3x/pmix/INSTALL b/opal/mca/pmix/pmix3x/pmix/INSTALL index 08fdfe641f..3fac5ad586 100644 --- a/opal/mca/pmix/pmix3x/pmix/INSTALL +++ b/opal/mca/pmix/pmix3x/pmix/INSTALL @@ -9,7 +9,7 @@ Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, Copyright (c) 2004-2005 The Regents of the University of California. All rights reserved. Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. -Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +Copyright (c) 2013-2019 Intel, Inc. All rights reserved. $COPYRIGHT$ Additional copyrights may follow @@ -22,9 +22,9 @@ For More Information This file is a *very* short overview of building and installing the PMIx library. Much more information is available in the -FAQ section on the PMIx web site: +How-To section on the PMIx web site: - http://pmix.github.io/pmix/faq + https://pmix.org/support/how-to/ Developer Builds @@ -56,6 +56,24 @@ shell$ ./configure --prefix=/where/to/install [...lots of output...] shell$ make all install +NOTE: this version of PMIx requires the Libevent package to build +and operate. Any version of Libevent greater than or equal to +2.0.21 is acceptable. + +NOTE: this version of PMIx optionally supports the HWLOC package +for providing topology information to both the host environment +(by collecting local inventory for rollup) and local client +processes. Any version of HWLOC greater than 1.10 is supported, +although versions in the 2.x series are recommended. + +Note that you must point configure at the libevent installation +using the --with-libevent= option if it is in a non-standard +location. Similarly, non-standard locations for the HWLOC package +must be specified using the --with-hwloc= option. In both +cases, PMIx will automatically detect these packages in standard +locations and build-in support for them unless otherwise specified +using the respective configure option. + If you need special access to install, then you can execute "make all" as a user with write permissions in the build tree, and a separate "make install" as a user with write permissions to the diff --git a/opal/mca/pmix/pmix3x/pmix/LICENSE b/opal/mca/pmix/pmix3x/pmix/LICENSE index 06f1248793..3eb0a094e0 100644 --- a/opal/mca/pmix/pmix3x/pmix/LICENSE +++ b/opal/mca/pmix/pmix3x/pmix/LICENSE @@ -26,7 +26,7 @@ Copyright (c) 2006-2010 Sun Microsystems, Inc. All rights reserved. Copyright (c) 2006-2010 The University of Houston. All rights reserved. Copyright (c) 2006-2009 Myricom, Inc. All rights reserved. Copyright (c) 2007-2008 UT-Battelle, LLC. All rights reserved. -Copyright (c) 2007-2018 IBM Corporation. All rights reserved. +Copyright (c) 2007-2019 IBM Corporation. All rights reserved. Copyright (c) 1998-2005 Forschungszentrum Juelich, Juelich Supercomputing Centre, Federal Republic of Germany Copyright (c) 2005-2008 ZIH, TU Dresden, Federal Republic of Germany @@ -36,7 +36,7 @@ Copyright (c) 2008-2009 Institut National de Recherche en Informatique. All rights reserved. Copyright (c) 2007 Lawrence Livermore National Security, LLC. All rights reserved. -Copyright (c) 2007-2018 Mellanox Technologies. All rights reserved. +Copyright (c) 2007-2019 Mellanox Technologies. All rights reserved. Copyright (c) 2006-2010 QLogic Corporation. All rights reserved. Copyright (c) 2008-2010 Oak Ridge National Labs. All rights reserved. Copyright (c) 2006-2010 Oracle and/or its affiliates. All rights reserved. @@ -45,7 +45,7 @@ Copyright (c) 2010 ARM ltd. All rights reserved. Copyright (c) 2010-2011 Alex Brick . All rights reserved. Copyright (c) 2012 The University of Wisconsin-La Crosse. All rights reserved. -Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +Copyright (c) 2013-2019 Intel, Inc. All rights reserved. Copyright (c) 2011-2014 NVIDIA Corporation. All rights reserved. $COPYRIGHT$ diff --git a/opal/mca/pmix/pmix3x/pmix/Makefile.am b/opal/mca/pmix/pmix3x/pmix/Makefile.am index 9d1013c08c..78fdedaafc 100644 --- a/opal/mca/pmix/pmix3x/pmix/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/Makefile.am @@ -37,10 +37,6 @@ dist_pmixdata_DATA = if ! PMIX_EMBEDDED_MODE dist_pmixdata_DATA += contrib/pmix-valgrind.supp -if PMIX_HAVE_PANDOC -SUBDIRS += man -endif - endif if PMIX_TESTS_EXAMPLES diff --git a/opal/mca/pmix/pmix3x/pmix/NEWS b/opal/mca/pmix/pmix3x/pmix/NEWS index 4ca89d8505..abc7978052 100644 --- a/opal/mca/pmix/pmix3x/pmix/NEWS +++ b/opal/mca/pmix/pmix3x/pmix/NEWS @@ -1,4 +1,4 @@ -Copyright (c) 2015-2018 Intel, Inc. All rights reserved. +Copyright (c) 2015-2019 Intel, Inc. All rights reserved. Copyright (c) 2017 IBM Corporation. All rights reserved. $COPYRIGHT$ @@ -17,8 +17,61 @@ the README file, PMIx typically maintains two separate version series simultaneously - the current release and one that is locked to only bug fixes. Since these series are semi-independent of each other, a single NEWS-worthy item might apply to different series. For -example, a bug might be fixed in the master, and then moved to the -current release as well as the "stable" bug fix release branch. +example, a bug might be fixed in the master, and then moved to +multiple release branches. + + +3.1.2 -- 24 Jan 2019 +---------------------- + - Fix a bug in macro identifying system events + - Restore some non-standard macros to the pmix_extend.h + header - these are considered "deprecated" and will be + removed from public-facing headers in future releases + + +3.1.1 -- 18 Jan 2019 +---------------------- +- Fix a bug in registration of default event handlers + that somehow slipped thru testing + + +3.1.0 -- 17 Jan 2019 +---------------------- +**** THIS RELEASE MARKS THE STARTING POINT FOR FULL COMPLIANCE +**** WITH THE PMIX v3 STANDARD. ALL API BEHAVIORS AND ATTRIBUTE +**** DEFINITIONS MEET THE v3 STANDARD SPECIFICATIONS. + - Add a new, faster dstore GDS component 'ds21' + - Performance optimizations for the dstore GDS components. + - Plug miscellaneous memory leaks + - Silence an unnecessary warning message when checking connection + to a non-supporting server + - Ensure lost-connection events get delivered to default event + handlers + - Correctly handle cache refresh for queries + - Protect against race conditions between host and internal library + when dealing with async requests + - Cleanup tool operations and add support for connections to + remote servers. Initial support for debugger direct/indirect + launch verified with PRRTE. Cleanup setting of tmpdir options. + Drop rendezvous files when acting as a launcher + - Automatically store the server URI for easy access by client + - Provide MCA parameter to control TCP connect retry/timeout + - Update event notification system to properly evict oldest events + when more space is needed + - Fix a number of error paths + - Update IOF cache code to properly drop oldest message. Provide + MCA parameter for setting cache size. + - Handle setsockopt(SO_RCVTIMEO) not being supported + - Ensure that epilogs get run even when connections unexpectedly + terminate. Properly split epilog strings to process multiple + paths + - Pass the tool's command line to the server so it can be returned + in queries + - Add support for C11 atomics + - Support collection and forwarding of fabric-specific envars + - Improve handling of hwloc configure option + - Fix PMIx_server_generate_regex to preserve node ordering + - Fix a bug when registering default event handlers 3.0.2 -- 18 Sept 2018 @@ -89,6 +142,46 @@ current release as well as the "stable" bug fix release branch. - Fix several memory and file descriptor leaks +2.2.2 -- 24 Jan 2019 +---------------------- + - Fix a bug in macro identifying system events + + +2.2.1 -- 18 Jan 2019 +---------------------- + - Fix a bug in registration of default event handlers + that somehow slipped thru testing + + +2.2.0 -- 17 Jan 2019 +---------------------- +**** THIS RELEASE MARKS THE STARTING POINT FOR FULL COMPLIANCE +**** WITH THE PMIX v2.2 STANDARD. ALL API BEHAVIORS AND ATTRIBUTE +**** DEFINITIONS MEET THE v2.2 STANDARD SPECIFICATIONS. + - Add a new, faster dstore GDS component 'ds21' + - Performance optimizations for the dstore GDS components. + - Plug miscellaneous memory leaks + - Silence an unnecessary warning message when checking connection + to a non-supporting server + - Ensure lost-connection events get delivered to default event + handlers + - Correctly handle cache refresh for queries + - Protect against race conditions between host and internal library + when dealing with async requests + - Cleanup tool operations and add support for connections to + remote servers. + - Automatically store the server URI for easy access by client + - Provide MCA parameter to control TCP connect retry/timeout + - Update event notification system to properly evict oldest events + when more space is needed + - Fix a number of error paths + - Handle setsockopt(SO_RCVTIMEO) not being supported + - Pass the tool's command line to the server so it can be returned + in queries + - Add support for C11 atomics + - Fix a bug when registering default event handlers + + 2.1.4 -- 18 Sep 2018 ---------------------- - Updated configury to silence warnings on older compilers diff --git a/opal/mca/pmix/pmix3x/pmix/README b/opal/mca/pmix/pmix3x/pmix/README index 22b0ec55bb..fa96176b7e 100644 --- a/opal/mca/pmix/pmix3x/pmix/README +++ b/opal/mca/pmix/pmix3x/pmix/README @@ -43,6 +43,11 @@ joe@mycomputer.example.com!). You can subscribe to the list here: https://groups.google.com/d/forum/pmix +Finally, just to round out all the possible ways to communicate with the +PMIx community, you are invited to join the community's Slack channel: + + pmix-workspace.slack.com + Thanks for your time. =========================================================================== @@ -192,21 +197,22 @@ PMIx's version numbers are the union of several different values: major, minor, release, and an optional quantifier. * Major: The major number is the first integer in the version string - (e.g., v1.2.3). Changes in the major number typically indicate a - significant change in the code base and/or end-user - functionality. The major number is always included in the version - number. + (e.g., v1.2.3) and indicates the corresponding version of the PMIx + Standard. In other words, a PMIx library release starting with "v2" + indicates that the implementation conforms to version 2 of the PMIx + Standard. * Minor: The minor number is the second integer in the version string (e.g., v1.2.3). Changes in the minor number typically indicate a incremental change in the code base and/or end-user - functionality. The minor number is always included in the version - number: + functionality, but not the supported version of the Standard. + The minor number is always included in the version number. * Release: The release number is the third integer in the version string (e.g., v1.2.3). Changes in the release number typically indicate a bug fix in the code base and/or end-user - functionality. + functionality. The release number is always included in the + version number. * Quantifier: PMIx version numbers sometimes have an arbitrary string affixed to the end of the version number. Common strings @@ -293,16 +299,15 @@ Application Binary Interface (ABI) Compatibility ------------------------------------------------ PMIx provides forward ABI compatibility in all versions of a given -feature release series and its corresponding -super stable series. For example, on a single platform, an pmix +feature release series. For example, on a single platform, an pmix application linked against PMIx v1.3.2 shared libraries can be updated to point to the shared libraries in any successive v1.3.x or v1.4 release and still work properly (e.g., via the LD_LIBRARY_PATH environment variable or other operating system mechanism). -PMIx reserves the right to break ABI compatibility at new feature +PMIx reserves the right to break ABI compatibility at new major release series. For example, the same pmix application from above -(linked against PMIx v1.3.2 shared libraries) will *not* work with +(linked against PMIx v1.3.2 shared libraries) may *not* work with PMIx v1.5 shared libraries. =========================================================================== diff --git a/opal/mca/pmix/pmix3x/pmix/VERSION b/opal/mca/pmix/pmix3x/pmix/VERSION index c10266b70b..a81e5274f8 100644 --- a/opal/mca/pmix/pmix3x/pmix/VERSION +++ b/opal/mca/pmix/pmix3x/pmix/VERSION @@ -14,7 +14,7 @@ # ... major=3 -minor=0 +minor=1 release=2 # greek is used for alpha or beta release tags. If it is non-empty, @@ -30,7 +30,7 @@ greek= # command, or with the date (if "git describe" fails) in the form of # "date". -repo_rev=gite574b10d +repo_rev=git0f7075f7 # If tarball_version is not empty, it is used as the version string in # the tarball filename, regardless of all other versions listed in @@ -44,7 +44,7 @@ tarball_version= # The date when this release was created -date="Sep 18, 2018" +date="Jan 24, 2019" # The shared library version of each of PMIx's public libraries. # These versions are maintained in accordance with the "Library @@ -75,6 +75,13 @@ date="Sep 18, 2018" # Version numbers are described in the Libtool current:revision:age # format. -libpmix_so_version=4:2:2 +libpmix_so_version=4:22:2 libpmi_so_version=1:0:0 libpmi2_so_version=1:0:0 + +# "Common" components install standalone libraries that are run-time +# # linked by one or more components. So they need to be versioned as +# # well. Yuck; this somewhat breaks the +# # components-don't-affect-the-build-system abstraction. +# +libmca_common_dstore_so_version=1:0:0 diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix.m4 index aeca068068..5d1660649f 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix.m4 @@ -17,7 +17,7 @@ dnl Copyright (c) 2009 Los Alamos National Security, LLC. All rights dnl reserved. dnl Copyright (c) 2009-2011 Oak Ridge National Labs. All rights reserved. dnl Copyright (c) 2011-2013 NVIDIA Corporation. All rights reserved. -dnl Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +dnl Copyright (c) 2013-2019 Intel, Inc. All rights reserved. dnl Copyright (c) 2015-2017 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. dnl Copyright (c) 2016 Mellanox Technologies, Inc. @@ -189,12 +189,36 @@ AC_DEFUN([PMIX_SETUP_CORE],[ [Link the output PMIx library to this extra lib (used in embedded mode)])) AC_MSG_CHECKING([for extra lib]) AS_IF([test ! -z "$with_pmix_extra_lib"], - [AC_MSG_RESULT([$with_pmix_extra_lib]) - PMIX_EXTRA_LIB=$with_pmix_extra_lib], + [AS_IF([test "$with_pmix_extra_lib" == "yes" || test "$with_pmix_extra_lib" == "no"], + [AC_MSG_RESULT([ERROR]) + AC_MSG_WARN([Invalid value for --with-extra-pmix-lib:]) + AC_MSG_WARN([ $with_pmix_extra_lib]) + AC_MSG_WARN([Must be path name of the library to add]) + AC_MSG_ERROR([Cannot continue])], + [AC_MSG_RESULT([$with_pmix_extra_lib]) + PMIX_EXTRA_LIB=$with_pmix_extra_lib])], [AC_MSG_RESULT([no]) PMIX_EXTRA_LIB=]) AC_SUBST(PMIX_EXTRA_LIB) + # Add any extra libtool lib? + AC_ARG_WITH([pmix-extra-ltlib], + AC_HELP_STRING([--with-pmix-extra-ltlib=LIB], + [Link any embedded components/tools that require it to the provided libtool lib (used in embedded mode)])) + AC_MSG_CHECKING([for extra ltlib]) + AS_IF([test ! -z "$with_pmix_extra_ltlib"], + [AS_IF([test "$with_pmix_extra_ltlib" == "yes" || test "$with_pmix_extra_ltlib" == "no"], + [AC_MSG_RESULT([ERROR]) + AC_MSG_WARN([Invalid value for --with-pmix-extra-ltlib:]) + AC_MSG_WARN([ $with_pmix_extra_ltlib]) + AC_MSG_WARN([Must be path name of the library to add]) + AC_MSG_ERROR([Cannot continue])], + [AC_MSG_RESULT([$with_pmix_extra_ltlib]) + PMIX_EXTRA_LTLIB=$with_pmix_extra_ltlib])], + [AC_MSG_RESULT([no]) + PMIX_EXTRA_LTLIB=]) + AC_SUBST(PMIX_EXTRA_LTLIB) + # # Package/brand string # @@ -391,7 +415,8 @@ AC_DEFUN([PMIX_SETUP_CORE],[ crt_externs.h signal.h \ ioLib.h sockLib.h hostLib.h limits.h \ sys/statfs.h sys/statvfs.h \ - netdb.h ucred.h zlib.h]) + netdb.h ucred.h zlib.h sys/auxv.h \ + sys/sysctl.h]) AC_CHECK_HEADERS([sys/mount.h], [], [], [AC_INCLUDES_DEFAULT @@ -655,10 +680,6 @@ AC_DEFUN([PMIX_SETUP_CORE],[ [AC_DEFINE_UNQUOTED([HAVE_UNIX_BYTESWAP], [1], [whether unix byteswap routines -- htonl, htons, nothl, ntohs -- are available])]) - # check pandoc separately so we can setup an AM_CONDITIONAL off it - AC_CHECK_PROG([pmix_have_pandoc], [pandoc], [yes], [no]) - AM_CONDITIONAL([PMIX_HAVE_PANDOC], [test "x$pmix_have_pandoc" = "xyes"]) - # # Make sure we can copy va_lists (need check declared, not linkable) # @@ -1078,20 +1099,6 @@ AC_DEFINE_UNQUOTED([PMIX_ENABLE_TIMING], [$WANT_PMIX_TIMING], [Whether we want developer-level timing support or not]) # -# Install header files -# -AC_MSG_CHECKING([if want to head developer-level header files]) -AC_ARG_WITH(devel-headers, - AC_HELP_STRING([--with-devel-headers], - [also install developer-level header files (only for internal PMIx developers, default: disabled)])) -if test "$with_devel_headers" = "yes"; then - AC_MSG_RESULT([yes]) - WANT_INSTALL_HEADERS=1 -else - AC_MSG_RESULT([no]) - WANT_INSTALL_HEADERS=0 -fi - # # Install backward compatibility support for PMI-1 and PMI-2 # diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_check_lock.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_check_lock.m4 index 69f184506c..0590dcf56c 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix_check_lock.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_check_lock.m4 @@ -57,4 +57,6 @@ AC_DEFUN([PMIX_CHECK_DSTOR_LOCK],[ fi fi LIBS="$orig_libs" + AM_CONDITIONAL([HAVE_DSTORE_PTHREAD_LOCK], [test "$_x_ac_pthread_lock_found" = "1"]) + AM_CONDITIONAL([HAVE_DSTORE_FCNTL_LOCK], [test "$_x_ac_fcntl_lock_found" = "1"]) ]) diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_check_os_flavors.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_check_os_flavors.m4 index 5c333ff511..fa08cf906c 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix_check_os_flavors.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_check_os_flavors.m4 @@ -1,7 +1,7 @@ dnl -*- shell-script -*- dnl dnl Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -dnl Copyright (c) 2014-2016 Intel, Inc. All rights reserved. +dnl Copyright (c) 2014-2019 Intel, Inc. All rights reserved. dnl Copyright (c) 2014 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. dnl @@ -57,6 +57,13 @@ AC_DEFUN([PMIX_CHECK_OS_FLAVORS], [$pmix_have_solaris], [Whether or not we have solaris]) + AS_IF([test "$pmix_found_apple" = "yes"], + [pmix_have_apple=1], + [pmix_have_apple=0]) + AC_DEFINE_UNQUOTED([PMIX_HAVE_APPLE], + [$pmix_have_apple], + [Whether or not we have apple]) + # check for sockaddr_in (a good sign we have TCP) AC_CHECK_HEADERS([netdb.h netinet/in.h netinet/tcp.h]) AC_CHECK_TYPES([struct sockaddr_in], diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_config_asm.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_config_asm.m4 index 062440499a..a46b73ad4d 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix_config_asm.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_config_asm.m4 @@ -2,22 +2,22 @@ dnl dnl Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana dnl University Research and Technology dnl Corporation. All rights reserved. -dnl Copyright (c) 2004-2015 The University of Tennessee and The University +dnl Copyright (c) 2004-2018 The University of Tennessee and The University dnl of Tennessee Research Foundation. All rights dnl reserved. dnl Copyright (c) 2004-2006 High Performance Computing Center Stuttgart, dnl University of Stuttgart. All rights reserved. dnl Copyright (c) 2004-2005 The Regents of the University of California. dnl All rights reserved. -dnl Copyright (c) 2008-2015 Cisco Systems, Inc. All rights reserved. +dnl Copyright (c) 2008-2018 Cisco Systems, Inc. All rights reserved. dnl Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. dnl Copyright (c) 2015-2017 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. -dnl Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights +dnl Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights dnl reserved. dnl Copyright (c) 2017 Amazon.com, Inc. or its affiliates. All Rights dnl reserved. -dnl Copyright (c) 2017 Intel, Inc. All rights reserved. +dnl Copyright (c) 2018-2019 Intel, Inc. All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -25,65 +25,262 @@ dnl dnl $HEADER$ dnl +dnl This is a C test to see if 128-bit __atomic_compare_exchange_n() +dnl actually works (e.g., it compiles and links successfully on +dnl ARM64+clang, but returns incorrect answers as of August 2018). +AC_DEFUN([PMIX_ATOMIC_COMPARE_EXCHANGE_N_TEST_SOURCE],[[ +#include +#include +#include + +typedef union { + uint64_t fake@<:@2@:>@; + __int128 real; +} pmix128; + +static void test1(void) +{ + // As of Aug 2018, we could not figure out a way to assign 128-bit + // constants -- the compilers would not accept it. So use a fake + // union to assign 2 uin64_t's to make a single __int128. + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }}; + pmix128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = __atomic_compare_exchange_n(&ptr.real, &expected.real, + desired.real, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + if ( !(r == false && ptr.real == expected.real)) { + exit(1); + } +} + +static void test2(void) +{ + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 expected = ptr; + pmix128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = __atomic_compare_exchange_n(&ptr.real, &expected.real, + desired.real, true, + __ATOMIC_RELAXED, __ATOMIC_RELAXED); + if (!(r == true && ptr.real == desired.real)) { + exit(2); + } +} + +int main(int argc, char** argv) +{ + test1(); + test2(); + return 0; +} +]]) + +dnl ------------------------------------------------------------------ + +dnl This is a C test to see if 128-bit __sync_bool_compare_and_swap() +dnl actually works (e.g., it compiles and links successfully on +dnl ARM64+clang, but returns incorrect answers as of August 2018). +AC_DEFUN([PMIX_SYNC_BOOL_COMPARE_AND_SWAP_TEST_SOURCE],[[ +#include +#include +#include + +typedef union { + uint64_t fake@<:@2@:>@; + __int128 real; +} pmix128; + +static void test1(void) +{ + // As of Aug 2018, we could not figure out a way to assign 128-bit + // constants -- the compilers would not accept it. So use a fake + // union to assign 2 uin64_t's to make a single __int128. + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 oldval = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }}; + pmix128 newval = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = __sync_bool_compare_and_swap(&ptr.real, oldval.real, newval.real); + if (!(r == false && ptr.real != newval.real)) { + exit(1); + } +} + +static void test2(void) +{ + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 oldval = ptr; + pmix128 newval = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = __sync_bool_compare_and_swap(&ptr.real, oldval.real, newval.real); + if (!(r == true && ptr.real == newval.real)) { + exit(2); + } +} + +int main(int argc, char** argv) +{ + test1(); + test2(); + return 0; +} +]]) + +dnl This is a C test to see if 128-bit __atomic_compare_exchange_n() +dnl actually works (e.g., it compiles and links successfully on +dnl ARM64+clang, but returns incorrect answers as of August 2018). +AC_DEFUN([PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE],[[ +#include +#include +#include +#include + +typedef union { + uint64_t fake@<:@2@:>@; + _Atomic __int128 real; +} pmix128; + +static void test1(void) +{ + // As of Aug 2018, we could not figure out a way to assign 128-bit + // constants -- the compilers would not accept it. So use a fake + // union to assign 2 uin64_t's to make a single __int128. + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }}; + pmix128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real, + desired.real, true, + atomic_relaxed, atomic_relaxed); + if ( !(r == false && ptr.real == expected.real)) { + exit(1); + } +} + +static void test2(void) +{ + pmix128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + pmix128 expected = ptr; + pmix128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real, + desired.real, true, + atomic_relaxed, atomic_relaxed); + if (!(r == true && ptr.real == desired.real)) { + exit(2); + } +} + +int main(int argc, char** argv) +{ + test1(); + test2(); + return 0; +} +]]) + +dnl ------------------------------------------------------------------ + +dnl +dnl Check to see if a specific function is linkable. +dnl +dnl Check with: +dnl 1. No compiler/linker flags. +dnl 2. CFLAGS += -mcx16 +dnl 3. LIBS += -latomic +dnl 4. Finally, if it links ok with any of #1, #2, or #3, actually try +dnl to run the test code (if we're not cross-compiling) and verify +dnl that it actually gives us the correct result. +dnl +dnl Note that we unfortunately can't use AC SEARCH_LIBS because its +dnl check incorrectly fails (because these functions are special compiler +dnl intrinsics -- SEARCH_LIBS tries with "check FUNC()", which the +dnl compiler complains doesn't match the internal prototype). So we have +dnl to use our own LINK_IFELSE tests. Indeed, since these functions are +dnl so special, we actually need a valid source code that calls the +dnl functions with correct arguments, etc. It's not enough, for example, +dnl to do the usual "try to set a function pointer to the symbol" trick to +dnl determine if these functions are available, because the compiler may +dnl not implement these as actual symbols. So just try to link a real +dnl test code. +dnl +dnl $1: function name to print +dnl $2: program to test +dnl $3: action if any of 1, 2, or 3 succeeds +dnl #4: action if all of 1, 2, and 3 fail +dnl +AC_DEFUN([PMIX_ASM_CHECK_ATOMIC_FUNC],[ + PMIX_VAR_SCOPE_PUSH([pmix_asm_check_func_happy pmix_asm_check_func_CFLAGS_save pmix_asm_check_func_LIBS_save]) + + pmix_asm_check_func_CFLAGS_save=$CFLAGS + pmix_asm_check_func_LIBS_save=$LIBS + + dnl Check with no compiler/linker flags + AC_MSG_CHECKING([for $1]) + AC_LINK_IFELSE([$2], + [pmix_asm_check_func_happy=1 + AC_MSG_RESULT([yes])], + [pmix_asm_check_func_happy=0 + AC_MSG_RESULT([no])]) + + dnl If that didn't work, try again with CFLAGS+=mcx16 + AS_IF([test $pmix_asm_check_func_happy -eq 0], + [AC_MSG_CHECKING([for $1 with -mcx16]) + CFLAGS="$CFLAGS -mcx16" + AC_LINK_IFELSE([$2], + [pmix_asm_check_func_happy=1 + AC_MSG_RESULT([yes])], + [pmix_asm_check_func_happy=0 + CFLAGS=$pmix_asm_check_func_CFLAGS_save + AC_MSG_RESULT([no])]) + ]) + + dnl If that didn't work, try again with LIBS+=-latomic + AS_IF([test $pmix_asm_check_func_happy -eq 0], + [AC_MSG_CHECKING([for $1 with -latomic]) + LIBS="$LIBS -latomic" + AC_LINK_IFELSE([$2], + [pmix_asm_check_func_happy=1 + AC_MSG_RESULT([yes])], + [pmix_asm_check_func_happy=0 + LIBS=$pmix_asm_check_func_LIBS_save + AC_MSG_RESULT([no])]) + ]) + + dnl If we have it, try it and make sure it gives a correct result. + dnl As of Aug 2018, we know that it links but does *not* work on clang + dnl 6 on ARM64. + AS_IF([test $pmix_asm_check_func_happy -eq 1], + [AC_MSG_CHECKING([if $1() gives correct results]) + AC_RUN_IFELSE([$2], + [AC_MSG_RESULT([yes])], + [pmix_asm_check_func_happy=0 + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) + + dnl If we were unsuccessful, restore CFLAGS/LIBS + AS_IF([test $pmix_asm_check_func_happy -eq 0], + [CFLAGS=$pmix_asm_check_func_CFLAGS_save + LIBS=$pmix_asm_check_func_LIBS_save]) + + dnl Run the user actions + AS_IF([test $pmix_asm_check_func_happy -eq 1], [$3], [$4]) + + PMIX_VAR_SCOPE_POP +]) + +dnl ------------------------------------------------------------------ AC_DEFUN([PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128], [ + PMIX_VAR_SCOPE_PUSH([sync_bool_compare_and_swap_128_result]) - PMIX_VAR_SCOPE_PUSH([sync_bool_compare_and_swap_128_result CFLAGS_save]) + # Do we have __sync_bool_compare_and_swap? + # Use a special macro because we need to check with a few different + # CFLAGS/LIBS. + PMIX_ASM_CHECK_ATOMIC_FUNC([__sync_bool_compare_and_swap], + [AC_LANG_SOURCE(PMIX_SYNC_BOOL_COMPARE_AND_SWAP_TEST_SOURCE)], + [sync_bool_compare_and_swap_128_result=1], + [sync_bool_compare_and_swap_128_result=0]) - AC_ARG_ENABLE([cross-cmpset128],[AC_HELP_STRING([--enable-cross-cmpset128], - [enable the use of the __sync builtin atomic compare-and-swap 128 when cross compiling])]) - - sync_bool_compare_and_swap_128_result=0 - - if test ! "$enable_cross_cmpset128" = "yes" ; then - AC_MSG_CHECKING([for processor support of __sync builtin atomic compare-and-swap on 128-bit values]) - - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);])], - [AC_MSG_RESULT([yes]) - sync_bool_compare_and_swap_128_result=1], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([no (cross compiling)])]) - - if test $sync_bool_compare_and_swap_128_result = 0 ; then - CFLAGS_save=$CFLAGS - CFLAGS="$CFLAGS -mcx16" - - AC_MSG_CHECKING([for __sync builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);])], - [AC_MSG_RESULT([yes]) - sync_bool_compare_and_swap_128_result=1 - CFLAGS_save="$CFLAGS"], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([no (cross compiling)])]) - - CFLAGS=$CFLAGS_save - fi - else - AC_MSG_CHECKING([for compiler support of __sync builtin atomic compare-and-swap on 128-bit values]) - - # Check if the compiler supports the __sync builtin - AC_TRY_LINK([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);], - [AC_MSG_RESULT([yes]) - sync_bool_compare_and_swap_128_result=1], - [AC_MSG_RESULT([no])]) - - if test $sync_bool_compare_and_swap_128_result = 0 ; then - CFLAGS_save=$CFLAGS - CFLAGS="$CFLAGS -mcx16" - - AC_MSG_CHECKING([for __sync builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) - AC_TRY_LINK([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);], - [AC_MSG_RESULT([yes]) - sync_bool_compare_and_swap_128_result=1 - CFLAGS_save="$CFLAGS"], - [AC_MSG_RESULT([no])]) - - CFLAGS=$CFLAGS_save - fi - fi - - AC_DEFINE_UNQUOTED([PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128], [$sync_bool_compare_and_swap_128_result], - [Whether the __sync builtin atomic compare and swap supports 128-bit values]) + AC_DEFINE_UNQUOTED([PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128], + [$sync_bool_compare_and_swap_128_result], + [Whether the __sync builtin atomic compare and swap supports 128-bit values]) PMIX_VAR_SCOPE_POP ]) @@ -112,7 +309,7 @@ __sync_add_and_fetch(&tmp, 1);], pmix_asm_sync_have_64bit=0]) AC_DEFINE_UNQUOTED([PMIX_ASM_SYNC_HAVE_64BIT],[$pmix_asm_sync_have_64bit], - [Whether 64-bit is supported by the __sync builtin atomics]) + [Whether 64-bit is supported by the __sync builtin atomics]) # Check for 128-bit support PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128 @@ -120,73 +317,110 @@ __sync_add_and_fetch(&tmp, 1);], AC_DEFUN([PMIX_CHECK_GCC_BUILTIN_CSWAP_INT128], [ + PMIX_VAR_SCOPE_PUSH([atomic_compare_exchange_n_128_result atomic_compare_exchange_n_128_CFLAGS_save atomic_compare_exchange_n_128_LIBS_save]) - PMIX_VAR_SCOPE_PUSH([atomic_compare_exchange_n_128_result CFLAGS_save]) + atomic_compare_exchange_n_128_CFLAGS_save=$CFLAGS + atomic_compare_exchange_n_128_LIBS_save=$LIBS - AC_ARG_ENABLE([cross-cmpset128],[AC_HELP_STRING([--enable-cross-cmpset128], - [enable the use of the __sync builtin atomic compare-and-swap 128 when cross compiling])]) + # Do we have __sync_bool_compare_and_swap? + # Use a special macro because we need to check with a few different + # CFLAGS/LIBS. + PMIX_ASM_CHECK_ATOMIC_FUNC([__atomic_compare_exchange_n], + [AC_LANG_SOURCE(PMIX_ATOMIC_COMPARE_EXCHANGE_N_TEST_SOURCE)], + [atomic_compare_exchange_n_128_result=1], + [atomic_compare_exchange_n_128_result=0]) - atomic_compare_exchange_n_128_result=0 - - if test ! "$enable_cross_cmpset128" = "yes" ; then - AC_MSG_CHECKING([for processor support of __atomic builtin atomic compare-and-swap on 128-bit values]) - - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);])], - [AC_MSG_RESULT([yes]) - atomic_compare_exchange_n_128_result=1], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([no (cross compiling)])]) - - if test $atomic_compare_exchange_n_128_result = 0 ; then - CFLAGS_save=$CFLAGS - CFLAGS="$CFLAGS -mcx16" - - AC_MSG_CHECKING([for __atomic builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);])], - [AC_MSG_RESULT([yes]) - atomic_compare_exchange_n_128_result=1 - CFLAGS_save="$CFLAGS"], - [AC_MSG_RESULT([no])], - [AC_MSG_RESULT([no (cross compiling)])]) - - CFLAGS=$CFLAGS_save - fi - - if test $atomic_compare_exchange_n_128_result = 1 ; then - AC_MSG_CHECKING([if __int128 atomic compare-and-swap is always lock-free]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([], [if (!__atomic_always_lock_free(16, 0)) { return 1; }])], + # If we have it and it works, check to make sure it is always lock + # free. + AS_IF([test $atomic_compare_exchange_n_128_result -eq 1], + [AC_MSG_CHECKING([if __int128 atomic compare-and-swap is always lock-free]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([], [if (!__atomic_always_lock_free(16, 0)) { return 1; }])], [AC_MSG_RESULT([yes])], - [AC_MSG_RESULT([no]) - PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128 - atomic_compare_exchange_n_128_result=0], - [AC_MSG_RESULT([no (cross compiling)])]) - fi - else - AC_MSG_CHECKING([for compiler support of __atomic builtin atomic compare-and-swap on 128-bit values]) + [atomic_compare_exchange_n_128_result=0 + # If this test fails, need to reset CFLAGS/LIBS (the + # above tests atomically set CFLAGS/LIBS or not; this + # test is running after the fact, so we have to undo + # the side-effects of setting CFLAGS/LIBS if the above + # tests passed). + CFLAGS=$atomic_compare_exchange_n_128_CFLAGS_save + LIBS=$atomic_compare_exchange_n_128_LIBS_save + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) - # Check if the compiler supports the __atomic builtin - AC_TRY_LINK([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);], - [AC_MSG_RESULT([yes]) - atomic_compare_exchange_n_128_result=1], - [AC_MSG_RESULT([no])]) + AC_DEFINE_UNQUOTED([PMIX_HAVE_GCC_BUILTIN_CSWAP_INT128], + [$atomic_compare_exchange_n_128_result], + [Whether the __atomic builtin atomic compare swap is both supported and lock-free on 128-bit values]) - if test $atomic_compare_exchange_n_128_result = 0 ; then - CFLAGS_save=$CFLAGS - CFLAGS="$CFLAGS -mcx16" + dnl If we could not find decent support for 128-bits __atomic let's + dnl try the GCC _sync + AS_IF([test $atomic_compare_exchange_n_128_result -eq 0], + [PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128]) - AC_MSG_CHECKING([for __atomic builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) - AC_TRY_LINK([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);], - [AC_MSG_RESULT([yes]) - atomic_compare_exchange_n_128_result=1 - CFLAGS_save="$CFLAGS"], - [AC_MSG_RESULT([no])]) + PMIX_VAR_SCOPE_POP +]) - CFLAGS=$CFLAGS_save - fi - fi +AC_DEFUN([PMIX_CHECK_GCC_ATOMIC_BUILTINS], [ + AC_MSG_CHECKING([for __atomic builtin atomics]) - AC_DEFINE_UNQUOTED([PMIX_HAVE_GCC_BUILTIN_CSWAP_INT128], [$atomic_compare_exchange_n_128_result], - [Whether the __atomic builtin atomic compare and swap is lock-free on 128-bit values]) + AC_TRY_LINK([ +#include +uint32_t tmp, old = 0; +uint64_t tmp64, old64 = 0;], [ +__atomic_thread_fence(__ATOMIC_SEQ_CST); +__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED); +__atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) + + # Check for 128-bit support + PMIX_CHECK_GCC_BUILTIN_CSWAP_INT128 +]) + +AC_DEFUN([PMIX_CHECK_C11_CSWAP_INT128], [ + PMIX_VAR_SCOPE_PUSH([atomic_compare_exchange_result atomic_compare_exchange_CFLAGS_save atomic_compare_exchange_LIBS_save]) + + atomic_compare_exchange_CFLAGS_save=$CFLAGS + atomic_compare_exchange_LIBS_save=$LIBS + + # Do we have C11 atomics on 128-bit integers? + # Use a special macro because we need to check with a few different + # CFLAGS/LIBS. + PMIX_ASM_CHECK_ATOMIC_FUNC([atomic_compare_exchange_strong_16], + [AC_LANG_SOURCE(PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE)], + [atomic_compare_exchange_result=1], + [atomic_compare_exchange_result=0]) + + # If we have it and it works, check to make sure it is always lock + # free. + AS_IF([test $atomic_compare_exchange_result -eq 1], + [AC_MSG_CHECKING([if C11 __int128 atomic compare-and-swap is always lock-free]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([#include ], [_Atomic __int128_t x; if (!atomic_is_lock_free(&x)) { return 1; }])], + [AC_MSG_RESULT([yes])], + [atomic_compare_exchange_result=0 + # If this test fails, need to reset CFLAGS/LIBS (the + # above tests atomically set CFLAGS/LIBS or not; this + # test is running after the fact, so we have to undo + # the side-effects of setting CFLAGS/LIBS if the above + # tests passed). + CFLAGS=$atomic_compare_exchange_CFLAGS_save + LIBS=$atomic_compare_exchange_LIBS_save + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) + + AC_DEFINE_UNQUOTED([PMIX_HAVE_C11_CSWAP_INT128], + [$atomic_compare_exchange_result], + [Whether C11 atomic compare swap is both supported and lock-free on 128-bit values]) + + dnl If we could not find decent support for 128-bits atomic let's + dnl try the GCC _sync + AS_IF([test $atomic_compare_exchange_result -eq 0], + [PMIX_CHECK_SYNC_BUILTIN_CSWAP_INT128]) PMIX_VAR_SCOPE_POP ]) @@ -533,7 +767,7 @@ dnl PMIX_CHECK_ASM_TYPE dnl dnl Sets PMIX_ASM_TYPE to the prefix for the function type to dnl set a symbol's type as function (needed on ELF for shared -dnl libaries). If no .type directive is needed, sets PMIX_ASM_TYPE +dnl libraries). If no .type directive is needed, sets PMIX_ASM_TYPE dnl to an empty string dnl dnl We look for @ \# % @@ -727,7 +961,7 @@ AC_DEFUN([PMIX_CHECK_SPARCV8PLUS],[ AC_MSG_CHECKING([if have Sparc v8+/v9 support]) sparc_result=0 PMIX_TRY_ASSEMBLE([$pmix_cv_asm_text - casa [%o0] 0x80, %o1, %o2], + casa [%o0] 0x80, %o1, %o2], [sparc_result=1], [sparc_result=0]) if test "$sparc_result" = "1" ; then @@ -746,35 +980,8 @@ dnl dnl PMIX_CHECK_CMPXCHG16B dnl dnl ################################################################# -AC_DEFUN([PMIX_CHECK_CMPXCHG16B],[ - PMIX_VAR_SCOPE_PUSH([cmpxchg16b_result]) - - AC_ARG_ENABLE([cross-cmpxchg16b],[AC_HELP_STRING([--enable-cross-cmpxchg16b], - [enable the use of the cmpxchg16b instruction when cross compiling])]) - - if test ! "$enable_cross_cmpxchg16b" = "yes" ; then - AC_MSG_CHECKING([if processor supports x86_64 16-byte compare-and-exchange]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([[unsigned char tmp[16];]],[[ - __asm__ __volatile__ ("lock cmpxchg16b (%%rsi)" : : "S" (tmp) : "memory", "cc");]])], - [AC_MSG_RESULT([yes]) - cmpxchg16b_result=1], - [AC_MSG_RESULT([no]) - cmpxchg16b_result=0], - [AC_MSG_RESULT([no (cross-compiling)]) - cmpxchg16b_result=0]) - else - AC_MSG_CHECKING([if assembler supports x86_64 16-byte compare-and-exchange]) - - PMIX_TRY_ASSEMBLE([$pmix_cv_asm_text - cmpxchg16b 0], - [AC_MSG_RESULT([yes]) - cmpxchg16b_result=1], - [AC_MSG_RESULT([no]) - cmpxchg16b_result=0]) - fi - if test "$cmpxchg16b_result" = 1; then - AC_MSG_CHECKING([if compiler correctly handles volatile 128bits]) - AC_RUN_IFELSE([AC_LANG_PROGRAM([#include +AC_DEFUN([PMIX_CMPXCHG16B_TEST_SOURCE],[[ +#include #include union pmix_counted_pointer_t { @@ -788,8 +995,10 @@ union pmix_counted_pointer_t { int128_t value; #endif }; -typedef union pmix_counted_pointer_t pmix_counted_pointer_t;], - [volatile pmix_counted_pointer_t a; +typedef union pmix_counted_pointer_t pmix_counted_pointer_t; + +int main(int argc, char* argv) { + volatile pmix_counted_pointer_t a; pmix_counted_pointer_t b; a.data.counter = 0; @@ -814,12 +1023,28 @@ typedef union pmix_counted_pointer_t pmix_counted_pointer_t;], return (a.value != b.value); #else return 0; -#endif])], - [AC_MSG_RESULT([yes])], - [AC_MSG_RESULT([no]) - cmpxchg16b_result=0], - [AC_MSG_RESULT([untested, assuming ok])]) - fi +#endif +} +]]) + +AC_DEFUN([PMIX_CHECK_CMPXCHG16B],[ + PMIX_VAR_SCOPE_PUSH([cmpxchg16b_result]) + + PMIX_ASM_CHECK_ATOMIC_FUNC([cmpxchg16b], + [AC_LANG_PROGRAM([[unsigned char tmp[16];]], + [[__asm__ __volatile__ ("lock cmpxchg16b (%%rsi)" : : "S" (tmp) : "memory", "cc");]])], + [cmpxchg16b_result=1], + [cmpxchg16b_result=0]) + # If we have it, make sure it works. + AS_IF([test $cmpxchg16b_result -eq 1], + [AC_MSG_CHECKING([if cmpxchg16b_result works]) + AC_RUN_IFELSE([AC_LANG_SOURCE(PMIX_CMPXCHG16B_TEST_SOURCE)], + [AC_MSG_RESULT([yes])], + [cmpxchg16b_result=0 + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) + AC_DEFINE_UNQUOTED([PMIX_HAVE_CMPXCHG16B], [$cmpxchg16b_result], [Whether the processor supports the cmpxchg16b instruction]) PMIX_VAR_SCOPE_POP @@ -832,7 +1057,7 @@ dnl dnl Check if the compiler is capable of doing GCC-style inline dnl assembly. Some compilers emit a warning and ignore the inline dnl assembly (xlc on OS X) and compile without error. Therefore, -dnl the test attempts to run the emited code to check that the +dnl the test attempts to run the emitted code to check that the dnl assembly is actually run. To run this test, one argument to dnl the macro must be an assembly instruction in gcc format to move dnl the value 0 into the register containing the variable ret. @@ -885,7 +1110,7 @@ return ret; if test "$asm_result" = "yes" ; then PMIX_C_GCC_INLINE_ASSEMBLY=1 - pmix_cv_asm_inline_supported="yes" + pmix_cv_asm_inline_supported="yes" else PMIX_C_GCC_INLINE_ASSEMBLY=0 fi @@ -912,18 +1137,30 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ AC_REQUIRE([PMIX_SETUP_CC]) AC_REQUIRE([AM_PROG_AS]) + AC_ARG_ENABLE([c11-atomics],[AC_HELP_STRING([--enable-c11-atomics], + [Enable use of C11 atomics if available (default: enabled)])]) + AC_ARG_ENABLE([builtin-atomics], [AC_HELP_STRING([--enable-builtin-atomics], - [Enable use of __sync builtin atomics (default: enabled)])], - [], [enable_builtin_atomics="yes"]) + [Enable use of __sync builtin atomics (default: disabled)])]) - pmix_cv_asm_builtin="BUILTIN_NO" - AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"], - [PMIX_CHECK_GCC_ATOMIC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_GCC"], [])]) - AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"], - [PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], [])]) - AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], - [AC_MSG_WARN([__sync builtin atomics requested but not found - proceeding with inline atomics])]) + PMIX_CHECK_C11_CSWAP_INT128 + + if test "x$enable_c11_atomics" != "xno" && test "$pmix_cv_c11_supported" = "yes" ; then + pmix_cv_asm_builtin="BUILTIN_C11" + PMIX_CHECK_C11_CSWAP_INT128 + elif test "x$enable_c11_atomics" = "xyes"; then + AC_MSG_WARN([C11 atomics were requested but are not supported]) + AC_MSG_ERROR([Cannot continue]) + else + pmix_cv_asm_builtin="BUILTIN_NO" + AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [PMIX_CHECK_GCC_ATOMIC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_GCC"], [])]) + AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], [])]) + AS_IF([test "$pmix_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [AC_MSG_ERROR([__sync builtin atomics requested but not found.])]) + fi PMIX_CHECK_ASM_PROC PMIX_CHECK_ASM_TEXT @@ -960,9 +1197,9 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ ia64-*) pmix_cv_asm_arch="IA64" PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], - [AC_MSG_ERROR([No atomic primitives available for $host])]) + [AC_MSG_ERROR([No atomic primitives available for $host])]) ;; - aarch64*) + aarch64*) pmix_cv_asm_arch="ARM64" PMIX_ASM_SUPPORT_64BIT=1 PMIX_ASM_ARM_VERSION=8 @@ -994,7 +1231,7 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ # uses Linux kernel helpers for some atomic operations pmix_cv_asm_arch="ARM" PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], - [AC_MSG_ERROR([No atomic primitives available for $host])]) + [AC_MSG_ERROR([No atomic primitives available for $host])]) ;; mips-*|mips64*) @@ -1002,7 +1239,7 @@ AC_DEFUN([PMIX_CONFIG_ASM],[ # a MIPS III machine (r4000 and later) pmix_cv_asm_arch="MIPS" PMIX_CHECK_SYNC_BUILTINS([pmix_cv_asm_builtin="BUILTIN_SYNC"], - [AC_MSG_ERROR([No atomic primitives available for $host])]) + [AC_MSG_ERROR([No atomic primitives available for $host])]) ;; powerpc-*|powerpc64-*|powerpcle-*|powerpc64le-*|rs6000-*|ppc-*) @@ -1070,11 +1307,11 @@ AC_MSG_ERROR([Can not continue.]) ;; esac - if test "x$PMIX_ASM_SUPPORT_64BIT" = "x1" && test "$pmix_cv_asm_builtin" = "BUILTIN_SYNC" && - test "$pmix_asm_sync_have_64bit" = "0" ; then - # __sync builtins exist but do not implement 64-bit support. Fall back on inline asm. - pmix_cv_asm_builtin="BUILTIN_NO" - fi + if test "x$PMIX_ASM_SUPPORT_64BIT" = "x1" && test "$pmix_cv_asm_builtin" = "BUILTIN_SYNC" && + test "$pmix_asm_sync_have_64bit" = "0" ; then + # __sync builtins exist but do not implement 64-bit support. Fall back on inline asm. + pmix_cv_asm_builtin="BUILTIN_NO" + fi if test "$pmix_cv_asm_builtin" = "BUILTIN_SYNC" || test "$pmix_cv_asm_builtin" = "BUILTIN_GCC" ; then AC_DEFINE([PMIX_C_GCC_INLINE_ASSEMBLY], [1], @@ -1097,7 +1334,7 @@ AC_MSG_ERROR([Can not continue.]) ;; esac - pmix_cv_asm_inline_supported="no" + pmix_cv_asm_inline_supported="no" # now that we know our architecture, try to inline assemble PMIX_CHECK_INLINE_C_GCC([$PMIX_GCC_INLINE_ASSIGN]) diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_mca.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_mca.m4 index 262e6f4a88..2978c928b9 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix_mca.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_mca.m4 @@ -11,7 +11,7 @@ dnl University of Stuttgart. All rights reserved. dnl Copyright (c) 2004-2005 The Regents of the University of California. dnl All rights reserved. dnl Copyright (c) 2010-2015 Cisco Systems, Inc. All rights reserved. -dnl Copyright (c) 2013-2016 Intel, Inc. All rights reserved. +dnl Copyright (c) 2013-2019 Intel, Inc. All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -428,7 +428,7 @@ AC_DEFUN([MCA_CONFIGURE_FRAMEWORK],[ # Create the final .h file that will be included in the type's # top-level glue. This lists all the static components. We don't # need to do this for "common". - if test "$2" != "common"; then + if test "$1" != "common"; then cat > $outfile <]],[[_Static_assert(sizeof(int64_t) == 8, "WTH");]]) - AS_IF([test $pmix_prog_cc_c11_helper__Thread_local_available -eq 1 && test $pmix_prog_cc_c11_helper_atomic_var_available -eq 1], + PMIX_CC_HELPER([if $CC $1 supports C11 atomic_fetch_xor_explicit], [pmix_prog_cc_c11_helper_atomic_fetch_xor_explicit_available], + [[#include +#include ]],[[_Atomic uint32_t a; uint32_t b; atomic_fetch_xor_explicit(&a, b, memory_order_relaxed);]]) + + + AS_IF([test $pmix_prog_cc_c11_helper__Thread_local_available -eq 1 && test $pmix_prog_cc_c11_helper_atomic_var_available -eq 1 && test $pmix_prog_cc_c11_helper_atomic_fetch_xor_explicit_available -eq 1], [$2], [$3]) @@ -128,7 +133,7 @@ AC_DEFUN([PMIX_SETUP_CC],[ AC_REQUIRE([_PMIX_PROG_CC]) AC_REQUIRE([AM_PROG_CC_C_O]) - PMIX_VAR_SCOPE_PUSH([pmix_prog_cc_c11_helper__Thread_local_available pmix_prog_cc_c11_helper_atomic_var_available pmix_prog_cc_c11_helper__Atomic_available pmix_prog_cc_c11_helper__static_assert_available pmix_prog_cc_c11_helper__Generic_available pmix_prog_cc__thread_available]) + PMIX_VAR_SCOPE_PUSH([pmix_prog_cc_c11_helper__Thread_local_available pmix_prog_cc_c11_helper_atomic_var_available pmix_prog_cc_c11_helper__Atomic_available pmix_prog_cc_c11_helper__static_assert_available pmix_prog_cc_c11_helper__Generic_available pmix_prog_cc__thread_available pmix_prog_cc_c11_helper_atomic_fetch_xor_explicit_available]) PMIX_PROG_CC_C11 diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_cli.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_cli.m4 index 9e66a4133f..043c0c4d44 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_cli.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_cli.m4 @@ -2,7 +2,8 @@ dnl -*- shell-script -*- dnl dnl Copyright (c) 2016 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. -dnl Copyright (c) 2017 Intel, Inc. All rights reserved. +dnl Copyright (c) 2017-2019 Intel, Inc. All rights reserved. +dnl Copyright (c) 2018 Cisco, Inc. All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -33,6 +34,7 @@ AC_DEFUN([PMIX_CAPTURE_CONFIGURE_CLI],[ eval "$1=\$$1\\ \$quoted_arg" done + AC_DEFINE_UNQUOTED([$1], ["$$1"], [Capture the configure cmd line]) PMIX_VAR_SCOPE_POP AC_SUBST($1) ]) diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4 index 86a9b9d447..8f6ed75176 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_hwloc.m4 @@ -2,7 +2,7 @@ # # Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -13,6 +13,43 @@ # MCA_hwloc_CONFIG([action-if-found], [action-if-not-found]) # -------------------------------------------------------------------- AC_DEFUN([PMIX_HWLOC_CONFIG],[ + AC_ARG_WITH([hwloc-header], + [AC_HELP_STRING([--with-hwloc-header=HEADER], + [The value that should be included in C files to include hwloc.h])]) + + AC_ARG_ENABLE([embedded-hwloc], + [AC_HELP_STRING([--enable-embedded-hwloc], + [Enable use of locally embedded hwloc])]) + + AS_IF([test "$enable_embedded_hwloc" = "yes"], + [_PMIX_HWLOC_EMBEDDED_MODE], + [_PMIX_HWLOC_EXTERNAL]) + + AC_MSG_CHECKING([hwloc header]) + AC_DEFINE_UNQUOTED([PMIX_HWLOC_HEADER], [$PMIX_HWLOC_HEADER], + [Location of hwloc.h]) + AC_MSG_RESULT([$PMIX_HWLOC_HEADER]) + + AC_DEFINE_UNQUOTED([PMIX_HAVE_HWLOC], [$pmix_hwloc_support], + [Whether or not we have hwloc support]) + + PMIX_SUMMARY_ADD([[External Packages]],[[HWLOC]], [pmix_hwloc], [$pmix_hwloc_support_will_build ($pmix_hwloc_source)]) +]) + +AC_DEFUN([_PMIX_HWLOC_EMBEDDED_MODE],[ + AC_MSG_CHECKING([for hwloc]) + AC_MSG_RESULT([assumed available (embedded mode)]) + + AS_IF([test -z "$with_hwloc_header" || test "$with_hwloc_header" = "yes"], + [PMIX_HWLOC_HEADER=""], + [PMIX_HWLOC_HEADER="$with_hwloc_header"]) + + pmix_hwloc_support=1 + pmix_hwloc_source=embedded + pmix_hwloc_support_will_build=yes + ]) + +AC_DEFUN([_PMIX_HWLOC_EXTERNAL],[ PMIX_VAR_SCOPE_PUSH([pmix_hwloc_dir pmix_hwloc_libdir pmix_hwloc_standard_lib_location pmix_hwloc_standard_header_location]) AC_ARG_WITH([hwloc], @@ -45,6 +82,17 @@ AC_DEFUN([PMIX_HWLOC_CONFIG],[ AC_MSG_RESULT([$pmix_hwloc_dir and $pmix_hwloc_libdir])], [AC_MSG_RESULT([$with_hwloc_libdir])]) else + pmix_hwloc_dir=/usr/include + if test -d /usr/lib; then + pmix_hwloc_libdir=/usr/lib + elif test -d /usr/lib64; then + pmix_hwloc_libdir=/usr/lib64 + else + AC_MSG_RESULT([not found]) + AC_MSG_WARN([Could not find /usr/lib or /usr/lib64 - you may]) + AC_MSG_WARN([need to specify --with-hwloc_libdir=]) + AC_MSG_ERROR([Can not continue]) + fi AC_MSG_RESULT([(default search paths)]) pmix_hwloc_standard_header_location=yes pmix_hwloc_standard_lib_location=yes @@ -62,18 +110,13 @@ AC_DEFUN([PMIX_HWLOC_CONFIG],[ [$pmix_hwloc_libdir], [pmix_hwloc_support=1], [pmix_hwloc_support=0]) - if test $pmix_hwloc_support = "1"; then - LIBS="$LIBS -lhwloc" - PMIX_EMBEDDED_LIBS="$PMIX_EMBEDDED_LIBS -lhwloc" - if test "$pmix_hwloc_standard_header_location" != "yes"; then - PMIX_EMBEDDED_CPPFLAGS="$PMIX_EMBEDDED_CPPFLAGS $pmix_hwloc_CPPFLAGS" - CPPFLAGS="$CPPFLAGS $pmix_hwloc_CPPFLAGS" - fi - if test "$pmix_hwloc_standard_lib_location" != "yes"; then - PMIX_EMBEDDED_LDFLAGS="$PMIX_EMBEDDED_LDFLAGS $pmix_hwloc_LDFLAGS" - LDFLAGS="$LDFLAGS $pmix_hwloc_LDFLAGS" - fi - fi + + AS_IF([test "$pmix_hwloc_standard_header_location" != "yes"], + [PMIX_FLAGS_APPEND_UNIQ(CPPFLAGS, $pmix_hwloc_CPPFLAGS)]) + + AS_IF([test "$pmix_hwloc_standard_lib_location" != "yes"], + [PMIX_FLAGS_APPEND_UNIQ(LDFLAGS, $pmix_hwloc_LDFLAGS)]) + PMIX_FLAGS_APPEND_UNIQ(LIBS, $pmix_hwloc_LIBS) fi if test ! -z "$with_hwloc" && test "$with_hwloc" != "no" && test "$pmix_hwloc_support" != "1"; then @@ -98,11 +141,16 @@ AC_DEFUN([PMIX_HWLOC_CONFIG],[ AC_MSG_CHECKING([will hwloc support be built]) if test "$pmix_hwloc_support" != "1"; then AC_MSG_RESULT([no]) + pmix_hwloc_source=none + pmix_hwloc_support_will_build=no else AC_MSG_RESULT([yes]) + pmix_hwloc_source=$pmix_hwloc_dir + pmix_hwloc_support_will_build=yes fi - AC_DEFINE_UNQUOTED([PMIX_HAVE_HWLOC], [$pmix_hwloc_support], - [Whether or not we have hwloc support]) + # Set output variables + PMIX_HWLOC_HEADER="" + PMIX_VAR_SCOPE_POP ])dnl diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_libevent.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_libevent.m4 index 739f0b8f22..2348a87aac 100644 --- a/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_libevent.m4 +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_setup_libevent.m4 @@ -2,7 +2,7 @@ # # Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -35,6 +35,8 @@ AC_DEFUN([PMIX_LIBEVENT_CONFIG],[ AC_DEFINE_UNQUOTED([PMIX_EVENT2_THREAD_HEADER], [$PMIX_EVENT2_THREAD_HEADER], [Location of event2/thread.h]) AC_MSG_RESULT([$PMIX_EVENT2_THREAD_HEADER]) + + PMIX_SUMMARY_ADD([[External Packages]],[[Libevent]], [pmix_libevent], [yes ($pmix_libevent_source)]) ]) AC_DEFUN([_PMIX_LIBEVENT_EMBEDDED_MODE],[ @@ -47,6 +49,7 @@ AC_DEFUN([_PMIX_LIBEVENT_EMBEDDED_MODE],[ [PMIX_EVENT_HEADER="$with_libevent_header" PMIX_EVENT2_THREAD_HEADER="$with_libevent_header"]) + pmix_libevent_source=embedded ]) AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[ @@ -115,9 +118,8 @@ AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[ AS_IF([test "$pmix_event_defaults" = "no"], [PMIX_FLAGS_APPEND_UNIQ(CPPFLAGS, $pmix_libevent_CPPFLAGS) - PMIX_FLAGS_APPEND_UNIQ(LIBS, $pmix_libevent_LIBS) PMIX_FLAGS_APPEND_UNIQ(LDFLAGS, $pmix_libevent_LDFLAGS)]) - + PMIX_FLAGS_APPEND_UNIQ(LIBS, $pmix_libevent_LIBS) # Ensure that this libevent has the symbol # "evthread_set_lock_callbacks", which will only exist if @@ -138,6 +140,7 @@ AC_DEFUN([_PMIX_LIBEVENT_EXTERNAL],[ # Set output variables PMIX_EVENT_HEADER="" PMIX_EVENT2_THREAD_HEADER="" + pmix_libevent_source=$pmix_event_dir PMIX_VAR_SCOPE_POP ])dnl diff --git a/opal/mca/pmix/pmix3x/pmix/config/pmix_summary.m4 b/opal/mca/pmix/pmix3x/pmix/config/pmix_summary.m4 new file mode 100644 index 0000000000..9b9bc9024e --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/config/pmix_summary.m4 @@ -0,0 +1,79 @@ +dnl -*- shell-script -*- +dnl +dnl Copyright (c) 2016 Los Alamos National Security, LLC. All rights +dnl reserved. +dnl Copyright (c) 2016-2018 Cisco Systems, Inc. All rights reserved +dnl Copyright (c) 2016 Research Organization for Information Science +dnl and Technology (RIST). All rights reserved. +dnl Copyright (c) 2018-2019 Intel, Inc. All rights reserved. +dnl $COPYRIGHT$ +dnl +dnl Additional copyrights may follow +dnl +dnl $HEADER$ +dnl +AC_DEFUN([PMIX_SUMMARY_ADD],[ + PMIX_VAR_SCOPE_PUSH([pmix_summary_section pmix_summary_line pmix_summary_section_current]) + + dnl need to replace spaces in the section name with somethis else. _ seems like a reasonable + dnl choice. if this changes remember to change PMIX_PRINT_SUMMARY as well. + pmix_summary_section=$(echo $1 | tr ' ' '_') + pmix_summary_line="$2: $4" + pmix_summary_section_current=$(eval echo \$pmix_summary_values_$pmix_summary_section) + + if test -z "$pmix_summary_section_current" ; then + if test -z "$pmix_summary_sections" ; then + pmix_summary_sections=$pmix_summary_section + else + pmix_summary_sections="$pmix_summary_sections $pmix_summary_section" + fi + eval pmix_summary_values_$pmix_summary_section=\"$pmix_summary_line\" + else + eval pmix_summary_values_$pmix_summary_section=\"$pmix_summary_section_current,$pmix_summary_line\" + fi + + PMIX_VAR_SCOPE_POP +]) + +AC_DEFUN([PMIX_SUMMARY_PRINT],[ + PMIX_VAR_SCOPE_PUSH([pmix_summary_section pmix_summary_section_name]) + cat < \$repo_arg, - "source-branch=s" => \$source_branch_arg, - "pages-branch=s" => \$pages_branch_arg, - "logfile-dir=s" => \$logfile_dir_arg, - "help|h" => \$help_arg, - "verbose" => \$verbose_arg, - ); - -if (!$ok || $help_arg) { - print "Invalid command line argument.\n\n" - if (!$ok); - print "Options: - --help | -h Print this message - --repo Git repo to be updated - --source-branch Branch containing source files (default: master) - --pages-branch Branch where man pages are to be output (default: gh-pages) - --logfile-dir Directory where execution log is to be written (default: /tmp) - --verbose Print debug info during execution\n"; - exit($ok ? 0 : 1); -} - -# Sanity checks -die "Must specify a git repo" - if (!defined($repo_arg)); - -##################################################################### - -my $logfile_dir = $logfile_dir_arg; -my $logfile_counter = 1; - -sub doit { - my $allowed_to_fail = shift; - my $cmd = shift; - my $stdout_file = shift; - - # Put a prefix on the logfiles so that we know that they belong to - # this script, and put a counter so that we know the sequence of - # logfiles - $stdout_file = "runall-md2nroff-$logfile_counter-$stdout_file"; - ++$logfile_counter; - - # Redirect stdout if requested - if (defined $stdout_file) { - $stdout_file = "$logfile_dir/$stdout_file.log"; - unlink($stdout_file); - $cmd .= " >$stdout_file"; - } elsif (!$verbose_arg && $cmd !~ />/) { - $cmd .= " >/dev/null"; - } - $cmd .= " 2>&1"; - - my $rc = system($cmd); - if (0 != $rc && !$allowed_to_fail) { - # If we die/fail, ensure to change out of the temp tree so - # that it can be removed upon exit. - chdir("/"); - die "Command $cmd failed: exit status $rc"; - } - - system("cat $stdout_file") - if ($verbose_arg && defined($stdout_file) && -f $stdout_file); -} - -sub verbose { - print @_ - if ($verbose_arg); -} - -##################################################################### - -# Setup a logfile dir just for this run -my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = - localtime(time); -$logfile_dir = - sprintf("%s/cron-run-all-md2nroff-logs-%04d-%02d-%02d-%02d%02d", - $logfile_dir_arg, $year + 1900, $mon + 1, $mday, - $hour, $min); -my $rc = system("mkdir $logfile_dir"); -if ($rc != 0 || ! -d $logfile_dir || ! -w $logfile_dir) { - chdir("/"); - die "mkdir of $logfile_dir failed, or can't write to it"; -} - -# First, git clone the source branch of the repo -verbose("*** Cloning repo: $repo_arg / $source_branch_arg...\n"); -my $tmpdir = File::Temp->newdir(); - -chdir($tmpdir); -doit(0, "git clone --single-branch --branch $source_branch_arg $repo_arg source", "git-clone"); - -# Next, git clone the pages branch of repo -if (defined($pages_branch_arg)) { - verbose("*** Cloning repo: $repo_arg / $pages_branch_arg...\n"); - doit(0, "git clone --single-branch --branch $pages_branch_arg $repo_arg pages", "git-clone2"); -} - -##################################################################### - -# Find all the *.\d.md files in the source repo -verbose("*** Finding markdown man pages...\n"); -opendir(DIR, "source/man"); -my @markdown_files = grep { /\.\d\.md$/ && -f "source/man/$_" } readdir(DIR); -closedir(DIR); -verbose("Found: @markdown_files\n"); - -##################################################################### - -# Copy each of the markdown files to the pages branch checkout -if (defined($pages_branch_arg)) { - chdir("pages/master"); - foreach my $file (@markdown_files) { - doit(0, "cp ../../source/man/$file man/$file", "loop-cp"); - - # Is there a new man page? If so, we need to "git add" it. - my $out = `git status --porcelain man/$file`; - doit(0, "git add man/$file", "loop-git-add") - if ($out =~ /^\?\?/); - } - - # Git commit those files in the pages repo and push them to the - # upstream repo so that they go live. If nothing changed, the commit - # and push will be no-ops. - chdir(".."); - doit(1, "git commit --no-verify -a -m \"Updated Markdown man pages from $source_branch_arg\"", - "git-commit-first"); - doit(1, "git push", "git-push-first"); -} - -##################################################################### - -# Now process each of the Markdown files in the source repo and -# generate new nroff man pages. -chdir("$tmpdir/source/man"); -foreach my $file (@markdown_files) { - doit(0, "../contrib/md2nroff.pl --source $file", "loop2-md2nroff"); - - # Did we generate a new man page? If so, we need to "git add" it. - my $man_file = basename($file); - - $man_file =~ m/\.(\d)\.md$/; - my $section = $1; - - $man_file =~ s/\.md$//; - - my $full_filename = "man$section/$man_file"; - - my $out = `git status --porcelain $full_filename`; - doit(0, "git add $full_filename", "loop2-git-add") - if ($out =~ /^\?\?/); -} - -# Similar to above: commit the newly-generated nroff pages and push -# them back upstream. If nothing changed, these will be no-ops. -doit(1, "git commit --no-verify -a -m \"Updated nroff-generated man pages\"", "git-commit-final"); -doit(1, "git push", "git-push-final"); - -# chdir out of the tmpdir so that it can be removed -chdir("/"); - -# If we get here, we finished successfully, so there's no need to keep -# the logfile dir around -system("rm -rf $logfile_dir"); - -exit(0); diff --git a/opal/mca/pmix/pmix3x/pmix/contrib/md2nroff.pl b/opal/mca/pmix/pmix3x/pmix/contrib/md2nroff.pl deleted file mode 100755 index 9eb33fe94b..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/contrib/md2nroff.pl +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env perl - -# Script to convert markdown to nroff man pages. -# -# The main conversion work is done via pandoc. But pandoc doesn't do -# everything exactly the way we want it, so use some perl regular -# expressions to fix up what pandoc doesn't get right. -# -# Do a "smart" write of the resulting output man page -- only write to -# the output file if the contents have actually changed compared to -# what was already there. - -use strict; -use warnings; - -use POSIX; -use File::Basename; -use Getopt::Long; -use File::Temp qw/tempfile/; - -my $source_arg; -my $target_arg; -my $help_arg; - -my $ok = Getopt::Long::GetOptions("source=s" => \$source_arg, - "target=s" => \$target_arg, - "help|h" => \$help_arg, - ); - -if ($help_arg) { - print "$0 --source input_MD_file --target output_nroff_file\n"; - exit(0); -} - -# Sanity checks -die "Must specify a source file" - if (!defined($source_arg)); -die "Source file does not exist ($source_arg)" - if (! -r $source_arg); - -my $pandoc = `which pandoc`; -die "Cannot find pandoc executable" - if ($pandoc eq ""); - -##################################################################### - -my $file = $source_arg; -$file =~ m/(\d+).md/; -my $section = $1; -die "Could not figure out the man page section: $source_arg" - if (!defined($section)); -my $shortfile = basename($file); -$shortfile =~ s/\.$section\.md$//; - -# If the target file was not specified, derive it from the source file -my $target; -if (!defined($target_arg)) { - $target_arg = $source_arg; - - $target_arg =~ m/\.(\d)\.md$/; - my $section = $1; - - my $dirname = dirname($target_arg); - my $basename = basename($target_arg); - $basename =~ s/\.md$//; - - $target = "$dirname/man$section/$basename"; -} else { - $target = $target_arg; -} - -print "*** Processing: $file -> $target\n"; - -# Read in the file -my $pandoc_input; -open(IN, $file) - || die "Can't open $file"; -$pandoc_input .= $_ - while (); -close(IN); - -# Remove the Jekyll header -$pandoc_input =~ s/.*---\n.+?---\n//s; - -# Remove the {% include ... %} directives -$pandoc_input =~ s/\n{0,1}\s*{%\s+include .+?\s+%}\s*\n/\n/g; - -# Change {% highlight c %} to ```c -$pandoc_input =~ s/^\s*{%\s+highlight\s+c\s+%}\s*$/\n```c/gmi; - -# Change {% endhighlight %} to ``` -$pandoc_input =~ s/^\s*\{\%\s+endhighlight\s+\%\}\s*$/```\n/gmi; - -# Pandoc does not handle markdown links in output nroff properly, -# so just remove all links. -while ($pandoc_input =~ m/\[(.+?)\]\(.+?\)/) { - my $text = $1; - $pandoc_input =~ s/\[(.+?)\]\(.+?\)/$text/; -} - -# Add the pandoc header -$pandoc_input = "% $shortfile($section) PMIx Programmer's Manual | \@VERSION\@ -% PMIx -% \@DATE\@\n\n$pandoc_input"; - -# Generate the nroff output -my ($fh, $temp_filename) = tempfile(); -print $fh $pandoc_input; -close($fh); - -open(IN, "pandoc -s --from=markdown --to=man $temp_filename|") - || die "Can't run pandoc"; -my $pandoc_nroff; -$pandoc_nroff .= $_ - while (); -close(IN); -unlink($temp_filename); - -# Now that we have the nroff string result, is it different than the -# target file? -my $write_nroff = 1; -if (-r $target) { - # If the target file exists, read it in - open(IN, $target) - || die "Can't open $target"; - my $target_nroff; - $target_nroff .= $_ - while (); - close(IN); - - # Remove the date from the target nroff string so that we can - # compare and ignore if the date has changed. Note that some - # versions of pandoc render dates as xxxx\-xx\-xx, and others - # render it as xxxx-xx-xx. Handle both. - $target_nroff =~ s/\"\d\d\d\d\\\-\d\d\\\-\d\d\"/\"\\\@DATE\\\@\"/; - $target_nroff =~ s/\"\d\d\d\d\-\d\d\-\d\d\"/\"\\\@DATE\\\@\"/; - - $write_nroff = 0 - if ($pandoc_nroff eq $target_nroff); -} - -# Do we need to write a new target nroff? -if ($write_nroff) { - - # What's the date right now? - my $now_string = strftime "%Y\\-%m\\-%d", localtime; - $pandoc_nroff =~ s/\\\@DATE\\\@/$now_string/g; - - # Make sure the target directory exists - my $dirname = dirname($target); - mkdir($dirname) - if (! -d $dirname); - - open(OUT, ">$target") - || die "Can't write to $target"; - print OUT $pandoc_nroff; - close(OUT); - - print "--> Wrote new $target\n"; -} else { - print "--> $target unchanged; not written\n"; -} - -exit(0); diff --git a/opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec b/opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec index b73d36ee39..ba7ec977ba 100644 --- a/opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec +++ b/opal/mca/pmix/pmix3x/pmix/contrib/pmix.spec @@ -192,7 +192,7 @@ Summary: An extended/exascale implementation of PMI Name: %{?_name:%{_name}}%{!?_name:pmix} -Version: 3.0.2 +Version: 3.1.2 Release: 1%{?dist} License: BSD Group: Development/Libraries diff --git a/opal/mca/pmix/pmix3x/pmix/examples/Makefile.am b/opal/mca/pmix/pmix3x/pmix/examples/Makefile.am index 72ed721613..b928ab2d58 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/examples/Makefile.am @@ -11,13 +11,14 @@ # All rights reserved. # Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2019 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow # # $HEADER$ # +headers = examples.h AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/include -I$(top_builddir)/include/pmix @@ -28,56 +29,56 @@ if !WANT_HIDDEN noinst_PROGRAMS += server endif -client_SOURCES = client.c +client_SOURCES = client.c examples.h client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) client_LDADD = $(top_builddir)/src/libpmix.la -client2_SOURCES = client2.c +client2_SOURCES = client2.c examples.h client2_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) client2_LDADD = $(top_builddir)/src/libpmix.la -debugger_SOURCES = debugger.c +debugger_SOURCES = debugger.c examples.h debugger_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) debugger_LDADD = $(top_builddir)/src/libpmix.la -debuggerd_SOURCES = debuggerd.c +debuggerd_SOURCES = debuggerd.c examples.h debuggerd_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) debuggerd_LDADD = $(top_builddir)/src/libpmix.la -alloc_SOURCES = alloc.c +alloc_SOURCES = alloc.c examples.h alloc_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) alloc_LDADD = $(top_builddir)/src/libpmix.la -jctrl_SOURCES = jctrl.c +jctrl_SOURCES = jctrl.c examples.h jctrl_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) jctrl_LDADD = $(top_builddir)/src/libpmix.la -dmodex_SOURCES = dmodex.c +dmodex_SOURCES = dmodex.c examples.h dmodex_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) dmodex_LDADD = $(top_builddir)/src/libpmix.la -dynamic_SOURCES = dynamic.c +dynamic_SOURCES = dynamic.c examples.h dynamic_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) dynamic_LDADD = $(top_builddir)/src/libpmix.la -fault_SOURCES = fault.c +fault_SOURCES = fault.c examples.h fault_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) fault_LDADD = $(top_builddir)/src/libpmix.la -pub_SOURCES = pub.c +pub_SOURCES = pub.c examples.h pub_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) pub_LDADD = $(top_builddir)/src/libpmix.la -pubi_SOURCES = pubi.c +pubi_SOURCES = pubi.c examples.h pubi_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) pubi_LDADD = $(top_builddir)/src/libpmix.la -tool_SOURCES = tool.c +tool_SOURCES = tool.c examples.h tool_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) tool_LDADD = $(top_builddir)/src/libpmix.la if !WANT_HIDDEN -server_SOURCES = server.c +server_SOURCES = server.c examples.h server_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) server_LDADD = $(top_builddir)/src/libpmix.la endif diff --git a/opal/mca/pmix/pmix3x/pmix/examples/alloc.c b/opal/mca/pmix/pmix3x/pmix/examples/alloc.c index f0cdf43a0e..6984c1e1e9 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/alloc.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/alloc.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -30,16 +30,7 @@ #include #include - -/* define a structure for collecting returned - * info from an allocation request */ -typedef struct { - volatile bool active; - pmix_info_t *info; - size_t ninfo; -} mydata_t; - -static volatile bool waiting_for_allocation = true; +#include "examples.h" /* this is a callback function for the PMIx_Query and * PMIx_Allocate APIs. The query will callback with a status indicating @@ -59,7 +50,7 @@ static void infocbfunc(pmix_status_t status, pmix_release_cbfunc_t release_fn, void *release_cbdata) { - mydata_t *mq = (mydata_t*)cbdata; + myquery_data_t *mq = (myquery_data_t*)cbdata; size_t n; fprintf(stderr, "Allocation request returned %s", PMIx_Error_string(status)); @@ -75,6 +66,9 @@ static void infocbfunc(pmix_status_t status, PMIX_INFO_XFER(&mq->info[n], &info[n]); } } + /* the status returned here indicates whether the requested + * information was found or not - preserve it */ + mq->lock.status = status; /* let the library release the data and cleanup from * the operation */ @@ -83,7 +77,7 @@ static void infocbfunc(pmix_status_t status, } /* release the block */ - mq->active = false; + DEBUG_WAKEUP_THREAD(&mq->lock); } /* this is an event notification function that we explicitly request @@ -100,12 +94,37 @@ static void release_fn(size_t evhdlr_registration_id, pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { + myrel_t *lock; + size_t n; + + /* find the return object */ + lock = NULL; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (myrel_t*)info[n].value.data.ptr; + break; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + fprintf(stderr, "LOCK WASN'T RETURNED IN RELEASE CALLBACK\n"); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + /* tell the event handler state machine that we are the last step */ if (NULL != cbfunc) { cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); } - /* flag that the allocation is complete so we can exit */ - waiting_for_allocation = false; + /* the status will be PMIX_ERR_ALLOC_COMPLETE since that is the code + * we registered to receive. The result of the allocation request is + * in the info array - for now, just assume success */ + lock->lock.status = PMIX_SUCCESS; + /* release the lock */ + DEBUG_WAKEUP_THREAD(&lock->lock); } /* event handler registration is done asynchronously because it @@ -114,18 +133,20 @@ static void release_fn(size_t evhdlr_registration_id, * the status of the request (success or an error), plus a numerical index * to the registered event. The index is used later on to deregister * an event handler - if we don't explicitly deregister it, then the - * PMIx server will do so when it see us exit */ + * PMIx server will do so when it sees us exit */ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + lock->evhandler_ref = evhandler_ref; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) @@ -138,11 +159,12 @@ int main(int argc, char **argv) uint32_t nprocs; pmix_info_t *info; uint64_t nnodes = 12; - mydata_t mydata; + myquery_data_t mydata; pmix_query_t *query; char *myallocation = "MYALLOCATION"; - volatile int active; - pmix_status_t code = PMIX_NOTIFY_ALLOC_COMPLETE; + mylock_t mylock; + pmix_status_t code; + myrel_t myrel; /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { @@ -163,63 +185,64 @@ int main(int argc, char **argv) PMIX_VALUE_RELEASE(val); fprintf(stderr, "Client %s:%d universe size %d\n", myproc.nspace, myproc.rank, nprocs); - /* initialize the return info struct */ - mydata.info = NULL; - mydata.ninfo = 0; - if (0 == myproc.rank) { /* try to get an allocation */ - mydata.active = true; + DEBUG_CONSTRUCT_MYQUERY(&mydata); PMIX_INFO_CREATE(info, 2); PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_NUM_NODES, &nnodes, PMIX_UINT64); PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_ID, myallocation, PMIX_STRING); - if (PMIX_SUCCESS != (rc = PMIx_Allocation_request_nb(PMIX_ALLOC_NEW, info, 2, infocbfunc, NULL))) { + if (PMIX_SUCCESS != (rc = PMIx_Allocation_request_nb(PMIX_ALLOC_NEW, info, 2, infocbfunc, &mydata))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Allocation_request_nb failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } - while (mydata.active) { - usleep(10); - } + DEBUG_WAIT_THREAD(&mydata.lock); PMIX_INFO_FREE(info, 2); - if (NULL != mydata.info) { - PMIX_INFO_FREE(mydata.info, mydata.ninfo); - } + fprintf(stderr, "Client ns %s rank %d: Allocation returned status: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(mydata.lock.status)); + DEBUG_DESTRUCT_MYQUERY(&mydata); + } else if (1 == myproc.rank) { - /* register a handler specifically for when the allocation - * operation completes */ - PMIX_INFO_CREATE(info, 1); + /* demonstrate a notification based approach - register a handler + * specifically for when the allocation operation completes */ + DEBUG_CONSTRUCT_MYREL(&myrel); + PMIX_INFO_CREATE(info, 2); PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_ID, myallocation, PMIX_STRING); - active = -1; - PMIx_Register_event_handler(&code, 1, info, 1, - release_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - usleep(10); - } - if (0 != active) { - exit(active); - } - PMIX_INFO_FREE(info, 1); + PMIX_INFO_LOAD(&info[1], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER); + DEBUG_CONSTRUCT_LOCK(&mylock); + code = PMIX_NOTIFY_ALLOC_COMPLETE; + PMIx_Register_event_handler(&code, 1, info, 2, + release_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + PMIX_INFO_FREE(info, 2); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + /* now wait to hear that the request is complete */ - while (waiting_for_allocation) { - usleep(10); - } + DEBUG_WAIT_THREAD(&myrel.lock); + fprintf(stderr, "[%s:%d] Allocation returned status: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(myrel.lock.status)); + DEBUG_DESTRUCT_MYREL(&myrel); + } else { - /* I am not the root rank, so let me wait a little while and then - * query the status of the allocation request */ + /* demonstrate a query-based approach - wait a little while and ask to + * see if it was done */ usleep(10); + DEBUG_CONSTRUCT_MYQUERY(&mydata); + PMIX_QUERY_CREATE(query, 1); PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_ALLOC_STATUS); PMIX_INFO_CREATE(query[0].qualifiers, 1); PMIX_INFO_LOAD(&query[0].qualifiers[0], PMIX_ALLOC_ID, myallocation, PMIX_STRING); - mydata.active = true; + if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, 1, infocbfunc, (void*)&mydata))) { fprintf(stderr, "PMIx_Query_info failed: %d\n", rc); goto done; } - while (mydata.active) { - usleep(10); - } + DEBUG_WAIT_THREAD(&mydata.lock); PMIX_QUERY_FREE(query, 1); + fprintf(stderr, "[%s:%d] Allocation returned status: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(mydata.lock.status)); + DEBUG_DESTRUCT_MYQUERY(&mydata); } done: diff --git a/opal/mca/pmix/pmix3x/pmix/examples/client.c b/opal/mca/pmix/pmix3x/pmix/examples/client.c index 7e38608950..519ef649d6 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/client.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/client.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -31,8 +31,8 @@ #include #include +#include "examples.h" -static volatile bool waiting_for_debugger = true; static pmix_proc_t myproc; /* this is the event notification function we pass down below @@ -66,10 +66,36 @@ static void release_fn(size_t evhdlr_registration_id, pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { + myrel_t *lock; + size_t n; + + /* find the return object */ + lock = NULL; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (myrel_t*)info[n].value.data.ptr; + break; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + fprintf(stderr, "LOCK WASN'T RETURNED IN RELEASE CALLBACK\n"); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + + /* tell the event handler state machine that we are the last step */ if (NULL != cbfunc) { cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); } - waiting_for_debugger = false; + /* the status will be PMIX_ERR_DEBUGGER_RELEASE since that is the code + * we registered to receive, so just return success */ + lock->lock.status = PMIX_SUCCESS; + /* release the lock */ + DEBUG_WAKEUP_THREAD(&lock->lock); } /* event handler registration is done asynchronously because it @@ -83,27 +109,34 @@ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + lock->evhandler_ref = evhandler_ref; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) { - int rc; + pmix_status_t rc; pmix_value_t value; pmix_value_t *val = &value; - char *tmp, *ptr, *p; + char *tmp; pmix_proc_t proc; - uint32_t nprocs, m, n, local_cnt, *localpeers; + uint32_t nprocs, n; pmix_info_t *info; - bool flag, local; - volatile int active; + bool flag; + mylock_t mylock; + myrel_t myrel; pmix_status_t dbg = PMIX_ERR_DEBUGGER_RELEASE; + pid_t pid; + + pid = getpid(); + fprintf(stderr, "Client %lu: Running\n", (unsigned long)pid); /* init us - note that the call to "init" includes the return of * any job-related info provided by the RM. This includes any @@ -114,20 +147,21 @@ int main(int argc, char **argv) fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank, rc); exit(0); } - fprintf(stderr, "Client ns %s rank %d: Running\n", myproc.nspace, myproc.rank); + fprintf(stderr, "Client ns %s rank %d pid %lu: Running\n", myproc.nspace, myproc.rank, (unsigned long)pid); /* register our default event handler - again, this isn't strictly * required, but is generally good practice */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - sleep(1); - } - if (0 != active) { + notification_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + + if (PMIX_SUCCESS != rc) { fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank); - exit(active); + goto done; } /* job-related info is found in our nspace, assigned to the @@ -145,21 +179,24 @@ int main(int argc, char **argv) * debugger */ if (PMIX_SUCCESS == (rc = PMIx_Get(&proc, PMIX_DEBUG_WAIT_FOR_NOTIFY, NULL, 0, &val))) { /* register for debugger release */ - active = -1; - PMIx_Register_event_handler(&dbg, 1, NULL, 0, - release_fn, evhandler_reg_callbk, (void*)&active); + DEBUG_CONSTRUCT_LOCK(&mylock); + PMIX_INFO_CREATE(info, 1); + DEBUG_CONSTRUCT_MYREL(&myrel); + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER); + PMIx_Register_event_handler(&dbg, 1, info, 1, + release_fn, evhandler_reg_callbk, (void*)&mylock); /* wait for registration to complete */ - while (-1 == active) { - sleep(1); - } - if (0 != active) { + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + PMIX_INFO_FREE(info, 1); + if (PMIX_SUCCESS != rc) { fprintf(stderr, "[%s:%d] Debug handler registration failed\n", myproc.nspace, myproc.rank); - exit(active); + goto done; } /* wait for debugger release */ - while (waiting_for_debugger) { - sleep(1); - } + DEBUG_WAIT_THREAD(&myrel.lock); + DEBUG_DESTRUCT_MYREL(&myrel); } /* get our universe size */ @@ -196,7 +233,7 @@ int main(int argc, char **argv) value.type = PMIX_UINT64; value.data.uint64 = 1234; if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_LOCAL, tmp, &value))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Put local failed: %d\n", myproc.nspace, myproc.rank, rc); + fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } free(tmp); @@ -207,7 +244,7 @@ int main(int argc, char **argv) value.type = PMIX_STRING; value.data.string = "1234"; if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_REMOTE, tmp, &value))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Put remote failed: %d\n", myproc.nspace, myproc.rank, rc); + fprintf(stderr, "Client ns %s rank %d: PMIx_Put internal failed: %d\n", myproc.nspace, myproc.rank, rc); goto done; } free(tmp); @@ -230,96 +267,53 @@ int main(int argc, char **argv) } PMIX_INFO_FREE(info, 1); - /* get the number of local peers */ - if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_LOCAL_SIZE, NULL, 0, &val))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_SIZE failed: %d", myproc.nspace, myproc.rank, rc); - goto done; - } - local_cnt = val->data.uint32; - PMIX_VALUE_RELEASE(val); - - /* create an array for the peers */ - localpeers = (uint32_t*)malloc(local_cnt * sizeof(int)); - - /* get the list of local peers */ - if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, PMIX_LOCAL_PEERS, NULL, 0, &val))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get PMIX_LOCAL_PEERS failed: %d", myproc.nspace, myproc.rank, rc); - goto done; - } - ptr = strdup(val->data.string); - PMIX_VALUE_RELEASE(val); - - /* populate the peers array */ - p = strtok(ptr, ","); - localpeers[0] = strtoul(p, NULL, 10); - for (n=1; n < local_cnt; n++) { - p = strtok(NULL, ","); - localpeers[n] = strtoul(p, NULL, 10); - } - free(ptr); - /* check the returned data */ for (n=0; n < nprocs; n++) { - if (n == myproc.rank) { - continue; + if (0 > asprintf(&tmp, "%s-%d-local", myproc.nspace, myproc.rank)) { + exit(1); } - proc.rank = n; - local = false; - for (m=0; m < local_cnt; m++) { - if (localpeers[m] == proc.rank) { - local = true; - break; - } + if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, tmp, NULL, 0, &val))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc); + goto done; } - if (local) { - if (0 > asprintf(&tmp, "%s-%d-local", proc.nspace, proc.rank)) { - exit(1); - } - if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, tmp, NULL, 0, &val))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc); - goto done; - } - if (PMIX_UINT64 != val->type) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type); - PMIX_VALUE_RELEASE(val); - free(tmp); - goto done; - } - if (1234 != val->data.uint64) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %d\n", myproc.nspace, myproc.rank, tmp, (int)val->data.uint64); - PMIX_VALUE_RELEASE(val); - free(tmp); - goto done; - } - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp); + if (PMIX_UINT64 != val->type) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type); PMIX_VALUE_RELEASE(val); free(tmp); - } else { - if (0 > asprintf(&tmp, "%s-%d-remote", proc.nspace, proc.rank)) { - exit(1); - } - if (PMIX_SUCCESS != (rc = PMIx_Get(&proc, tmp, NULL, 0, &val))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc); - goto done; - } - if (PMIX_STRING != val->type) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type); - PMIX_VALUE_RELEASE(val); - free(tmp); - goto done; - } - if (0 != strcmp(val->data.string, "1234")) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %s\n", myproc.nspace, myproc.rank, tmp, val->data.string); - PMIX_VALUE_RELEASE(val); - free(tmp); - goto done; - } - fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp); + goto done; + } + if (1234 != val->data.uint64) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %d\n", myproc.nspace, myproc.rank, tmp, (int)val->data.uint64); PMIX_VALUE_RELEASE(val); free(tmp); + goto done; } + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp); + PMIX_VALUE_RELEASE(val); + free(tmp); + if (0 > asprintf(&tmp, "%s-%d-remote", myproc.nspace, myproc.rank)) { + exit(1); + } + if (PMIX_SUCCESS != (rc = PMIx_Get(&myproc, tmp, NULL, 0, &val))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, myproc.rank, tmp, rc); + goto done; + } + if (PMIX_STRING != val->type) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong type: %d\n", myproc.nspace, myproc.rank, tmp, val->type); + PMIX_VALUE_RELEASE(val); + free(tmp); + goto done; + } + if (0 != strcmp(val->data.string, "1234")) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned wrong value: %s\n", myproc.nspace, myproc.rank, tmp, val->data.string); + PMIX_VALUE_RELEASE(val); + free(tmp); + goto done; + } + fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s returned correct\n", myproc.nspace, myproc.rank, tmp); + PMIX_VALUE_RELEASE(val); + free(tmp); } - free(localpeers); done: /* finalize us */ diff --git a/opal/mca/pmix/pmix3x/pmix/examples/client2.c b/opal/mca/pmix/pmix3x/pmix/examples/client2.c index 9b7f7e6c0f..4021c15ec7 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/client2.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/client2.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include #include +#include "examples.h" static pmix_proc_t myproc; @@ -62,25 +63,27 @@ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + lock->evhandler_ref = evhandler_ref; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) { - int rc; + pmix_status_t rc; pmix_value_t value; pmix_value_t *val, *vptr; pmix_proc_t proc; uint32_t nprocs, n, k; pmix_info_t *info; bool flag; - volatile int active; + mylock_t mylock; pmix_data_array_t da, *dptr; /* init us - note that the call to "init" includes the return of @@ -97,15 +100,16 @@ int main(int argc, char **argv) /* register our default event handler - again, this isn't strictly * required, but is generally good practice */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - sleep(1); - } - if (0 != active) { + notification_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + + if (PMIX_SUCCESS != rc) { fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank); - exit(active); + goto done; } /* job-related info is found in our nspace, assigned to the diff --git a/opal/mca/pmix/pmix3x/pmix/examples/dmodex.c b/opal/mca/pmix/pmix3x/pmix/examples/dmodex.c index c093d5bc82..76a1ac8ca0 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/dmodex.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/dmodex.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -33,24 +33,19 @@ #include #include +#include "examples.h" static uint32_t nprocs; static pmix_proc_t myproc; static uint32_t getcount = 0; -#define WAIT_FOR_COMPLETION(a) \ - do { \ - while ((a)) { \ - usleep(10); \ - } \ - } while (0) - static void opcbfunc(pmix_status_t status, void *cbdata) { - bool *active = (bool*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; fprintf(stderr, "%s:%d completed fence_nb\n", myproc.nspace, myproc.rank); - *active = false; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } static void valcbfunc(pmix_status_t status, @@ -98,7 +93,7 @@ int main(int argc, char **argv) char *tmp; pmix_proc_t proc; uint32_t n, num_gets; - bool active; + mylock_t mylock; /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { @@ -170,9 +165,10 @@ int main(int argc, char **argv) PMIX_PROC_CONSTRUCT(&proc); (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; - active = true; - if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(&proc, 1, NULL, 0, opcbfunc, &active))) { + DEBUG_CONSTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != (rc = PMIx_Fence_nb(&proc, 1, NULL, 0, opcbfunc, &mylock))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank, rc); + DEBUG_DESTRUCT_LOCK(&mylock); goto done; } @@ -203,7 +199,7 @@ int main(int argc, char **argv) } /* wait for the first fence to finish */ - WAIT_FOR_COMPLETION(active); + DEBUG_WAIT_THREAD(&mylock); /* wait for all my "get" calls to complete */ while (getcount < num_gets) { diff --git a/opal/mca/pmix/pmix3x/pmix/examples/dynamic.c b/opal/mca/pmix/pmix3x/pmix/examples/dynamic.c index 4ffe501f02..d7c0d3701f 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/dynamic.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/dynamic.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -35,7 +35,7 @@ #include #include - +#include "examples.h" static pmix_proc_t myproc; diff --git a/opal/mca/pmix/pmix3x/pmix/examples/examples.h b/opal/mca/pmix/pmix3x/pmix/examples/examples.h new file mode 100644 index 0000000000..d2e5ab7696 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/examples/examples.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include + +typedef struct { + pthread_mutex_t mutex; + pthread_cond_t cond; + volatile bool active; + pmix_status_t status; + int count; + size_t evhandler_ref; +} mylock_t; + +#define DEBUG_CONSTRUCT_LOCK(l) \ + do { \ + pthread_mutex_init(&(l)->mutex, NULL); \ + pthread_cond_init(&(l)->cond, NULL); \ + (l)->active = true; \ + (l)->status = PMIX_SUCCESS; \ + (l)->count = 0; \ + (l)->evhandler_ref = 0; \ + } while(0) + +#define DEBUG_DESTRUCT_LOCK(l) \ + do { \ + pthread_mutex_destroy(&(l)->mutex); \ + pthread_cond_destroy(&(l)->cond); \ + } while(0) + +#define DEBUG_WAIT_THREAD(lck) \ + do { \ + pthread_mutex_lock(&(lck)->mutex); \ + while ((lck)->active) { \ + pthread_cond_wait(&(lck)->cond, &(lck)->mutex); \ + } \ + pthread_mutex_unlock(&(lck)->mutex); \ + } while(0) + +#define DEBUG_WAKEUP_THREAD(lck) \ + do { \ + pthread_mutex_lock(&(lck)->mutex); \ + (lck)->active = false; \ + pthread_cond_broadcast(&(lck)->cond); \ + pthread_mutex_unlock(&(lck)->mutex); \ + } while(0) + +/* define a structure for collecting returned + * info from a query */ +typedef struct { + mylock_t lock; + pmix_info_t *info; + size_t ninfo; +} myquery_data_t; + +#define DEBUG_CONSTRUCT_MYQUERY(q) \ + do { \ + DEBUG_CONSTRUCT_LOCK(&((q)->lock)); \ + (q)->info = NULL; \ + (q)->ninfo = 0; \ + } while(0) + +#define DEBUG_DESTRUCT_MYQUERY(q) \ + do { \ + DEBUG_DESTRUCT_LOCK(&((q)->lock)); \ + if (NULL != (q)->info) { \ + PMIX_INFO_FREE((q)->info, (q)->ninfo); \ + } \ + } while(0) + +/* define a structure for releasing when a given + * nspace terminates */ +typedef struct { + mylock_t lock; + char *nspace; + int exit_code; + bool exit_code_given; +} myrel_t; + + +#define DEBUG_CONSTRUCT_MYREL(r) \ + do { \ + DEBUG_CONSTRUCT_LOCK(&((r)->lock)); \ + (r)->nspace = NULL; \ + (r)->exit_code = 0; \ + (r)->exit_code_given = false; \ + } while(0) + +#define DEBUG_DESTRUCT_MYREL(r) \ + do { \ + DEBUG_DESTRUCT_LOCK(&((r)->lock)); \ + if (NULL != (r)->nspace) { \ + free((r)->nspace); \ + } \ + } while(0) diff --git a/opal/mca/pmix/pmix3x/pmix/examples/fault.c b/opal/mca/pmix/pmix3x/pmix/examples/fault.c index a197f58961..abab388681 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/fault.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/fault.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -28,11 +28,12 @@ #include #include #include +#include #include +#include "examples.h" static pmix_proc_t myproc; -static bool completed; static void notification_fn(size_t evhdlr_registration_id, pmix_status_t status, @@ -42,22 +43,67 @@ static void notification_fn(size_t evhdlr_registration_id, pmix_event_notification_cbfunc_fn_t cbfunc, void *cbdata) { - fprintf(stderr, "Client %s:%d NOTIFIED with status %d\n", myproc.nspace, myproc.rank, status); - completed = true; + myrel_t *lock; + bool found; + int exit_code; + size_t n; + pmix_proc_t *affected = NULL; + + /* find our return object */ + lock = NULL; + found = false; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (myrel_t*)info[n].value.data.ptr; + /* not every RM will provide an exit code, but check if one was given */ + } else if (0 == strncmp(info[n].key, PMIX_EXIT_CODE, PMIX_MAX_KEYLEN)) { + exit_code = info[n].value.data.integer; + found = true; + } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + affected = info[n].value.data.proc; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + fprintf(stderr, "LOCK WASN'T RETURNED IN RELEASE CALLBACK\n"); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + + /* tell the event handler state machine that we are the last step */ + if (NULL != cbfunc) { + cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); + } + fprintf(stderr, "DEBUGGER DAEMON NOTIFIED TERMINATED - AFFECTED %s\n", + (NULL == affected) ? "NULL" : affected->nspace); + + if (found) { + lock->exit_code = exit_code; + lock->exit_code_given = true; + } + DEBUG_WAKEUP_THREAD(&lock->lock); } static void op_callbk(pmix_status_t status, void *cbdata) { + mylock_t *lock = (mylock_t*)cbdata; fprintf(stderr, "Client %s:%d OP CALLBACK CALLED WITH STATUS %d\n", myproc.nspace, myproc.rank, status); + DEBUG_WAKEUP_THREAD(lock); } -static void errhandler_reg_callbk(pmix_status_t status, +static void evhandler_reg_callbk(pmix_status_t status, size_t errhandler_ref, void *cbdata) { + mylock_t *lock = (mylock_t*)cbdata; + fprintf(stderr, "Client %s:%d ERRHANDLER REGISTRATION CALLBACK CALLED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)errhandler_ref); + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) @@ -67,6 +113,10 @@ int main(int argc, char **argv) pmix_value_t *val = &value; pmix_proc_t proc; uint32_t nprocs; + pmix_info_t *info; + mylock_t mylock; + myrel_t myrel; + pmix_status_t code[2] = {PMIX_ERR_PROC_ABORTED, PMIX_ERR_JOB_TERMINATED}; /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { @@ -87,11 +137,27 @@ int main(int argc, char **argv) nprocs = val->data.uint32; PMIX_VALUE_RELEASE(val); fprintf(stderr, "Client %s:%d universe size %d\n", myproc.nspace, myproc.rank, nprocs); - completed = false; - /* register our errhandler */ - PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, errhandler_reg_callbk, NULL); + /* register another handler specifically for when the target + * job completes */ + DEBUG_CONSTRUCT_MYREL(&myrel); + PMIX_INFO_CREATE(info, 2); + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER); + /* only call me back when one of us terminates */ + PMIX_INFO_LOAD(&info[1], PMIX_NSPACE, myproc.nspace, PMIX_STRING); + + DEBUG_CONSTRUCT_LOCK(&mylock); + PMIx_Register_event_handler(code, 2, info, 2, + notification_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + if (PMIX_SUCCESS != mylock.status) { + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + PMIX_INFO_FREE(info, 2); + goto done; + } + DEBUG_DESTRUCT_LOCK(&mylock); + PMIX_INFO_FREE(info, 2); /* call fence to sync */ PMIX_PROC_CONSTRUCT(&proc); @@ -109,17 +175,16 @@ int main(int argc, char **argv) exit(1); } /* everyone simply waits */ - while (!completed) { - struct timespec ts; - ts.tv_sec = 0; - ts.tv_nsec = 100000; - nanosleep(&ts, NULL); - } + DEBUG_WAIT_THREAD(&myrel.lock); + DEBUG_DESTRUCT_MYREL(&myrel); done: /* finalize us */ fprintf(stderr, "Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); - PMIx_Deregister_event_handler(1, op_callbk, NULL); + DEBUG_CONSTRUCT_LOCK(&mylock); + PMIx_Deregister_event_handler(1, op_callbk, &mylock); + DEBUG_WAIT_THREAD(&mylock); + DEBUG_DESTRUCT_LOCK(&mylock); if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) { fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %d\n", myproc.nspace, myproc.rank, rc); diff --git a/opal/mca/pmix/pmix3x/pmix/examples/jctrl.c b/opal/mca/pmix/pmix3x/pmix/examples/jctrl.c index 5c1c1d1f73..0dc244b3c6 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/jctrl.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/jctrl.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include #include +#include "examples.h" static pmix_proc_t myproc; @@ -63,13 +64,15 @@ static void evhandler_reg_callbk(pmix_status_t status, size_t evhandler_ref, void *cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; if (PMIX_SUCCESS != status) { fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref); } - *active = status; + lock->status = status; + lock->evhandler_ref = evhandler_ref; + DEBUG_WAKEUP_THREAD(lock); } static void infocbfunc(pmix_status_t status, @@ -78,26 +81,27 @@ static void infocbfunc(pmix_status_t status, pmix_release_cbfunc_t release_fn, void *release_cbdata) { - volatile int *active = (volatile int*)cbdata; + mylock_t *lock = (mylock_t*)cbdata; /* release the caller */ if (NULL != release_fn) { release_fn(release_cbdata); } - *active = status; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); } int main(int argc, char **argv) { - int rc; + pmix_status_t rc; pmix_value_t value; pmix_value_t *val = &value; pmix_proc_t proc; uint32_t nprocs, n; pmix_info_t *info, *iptr; bool flag; - volatile int active; + mylock_t mylock; pmix_data_array_t *dptr; /* init us - note that the call to "init" includes the return of @@ -111,15 +115,16 @@ int main(int argc, char **argv) /* register our default event handler - again, this isn't strictly * required, but is generally good practice */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); PMIx_Register_event_handler(NULL, 0, NULL, 0, - notification_fn, evhandler_reg_callbk, (void*)&active); - while (-1 == active) { - sleep(1); - } - if (0 != active) { + notification_fn, evhandler_reg_callbk, (void*)&mylock); + /* wait for registration to complete */ + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != rc) { fprintf(stderr, "[%s:%d] Default handler registration failed\n", myproc.nspace, myproc.rank); - exit(active); + goto done; } /* job-related info is found in our nspace, assigned to the @@ -145,12 +150,8 @@ int main(int argc, char **argv) PMIX_INFO_LOAD(&info[0], PMIX_JOB_CTRL_PREEMPTIBLE, (void*)&flag, PMIX_BOOL); /* can't use "load" to load a pmix_data_array_t */ (void)strncpy(info[1].key, PMIX_JOB_CTRL_CHECKPOINT_METHOD, PMIX_MAX_KEYLEN); - info[1].value.type = PMIX_DATA_ARRAY; - dptr = (pmix_data_array_t*)malloc(sizeof(pmix_data_array_t)); - info[1].value.data.darray = dptr; - dptr->type = PMIX_INFO; - dptr->size = 2; - PMIX_INFO_CREATE(dptr->array, dptr->size); + PMIX_DATA_ARRAY_CREATE(info[1].value.data.darray, 2, PMIX_INFO); + dptr = info[1].value.data.darray; rc = SIGUSR2; iptr = (pmix_info_t*)dptr->array; PMIX_INFO_LOAD(&iptr[0], PMIX_JOB_CTRL_CHECKPOINT_SIGNAL, &rc, PMIX_INT); @@ -159,18 +160,19 @@ int main(int argc, char **argv) /* since this is informational and not a requested operation, the target parameter * doesn't mean anything and can be ignored */ - active = -1; - if (PMIX_SUCCESS != (rc = PMIx_Job_control_nb(NULL, 0, info, 2, infocbfunc, (void*)&active))) { + DEBUG_CONSTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != (rc = PMIx_Job_control_nb(NULL, 0, info, 2, infocbfunc, (void*)&mylock))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Job_control_nb failed: %d\n", myproc.nspace, myproc.rank, rc); + DEBUG_DESTRUCT_LOCK(&mylock); goto done; } - while (-1 == active) { - sleep(1); - } + DEBUG_WAIT_THREAD(&mylock); PMIX_INFO_FREE(info, 2); - if (0 != active) { + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != rc) { fprintf(stderr, "Client ns %s rank %d: PMIx_Job_control_nb failed: %d\n", myproc.nspace, myproc.rank, rc); - exit(active); + goto done; } /* now request that this process be monitored using heartbeats */ @@ -185,20 +187,21 @@ int main(int argc, char **argv) PMIX_INFO_LOAD(&info[2], PMIX_MONITOR_HEARTBEAT_DROPS, &n, PMIX_UINT32); /* make the request */ - active = -1; + DEBUG_CONSTRUCT_LOCK(&mylock); if (PMIX_SUCCESS != (rc = PMIx_Process_monitor_nb(iptr, PMIX_MONITOR_HEARTBEAT_ALERT, - info, 3, infocbfunc, (void*)&active))) { + info, 3, infocbfunc, (void*)&mylock))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Process_monitor_nb failed: %d\n", myproc.nspace, myproc.rank, rc); + DEBUG_DESTRUCT_LOCK(&mylock); goto done; } - while (-1 == active) { - sleep(1); - } + DEBUG_WAIT_THREAD(&mylock); PMIX_INFO_FREE(iptr, 1); PMIX_INFO_FREE(info, 3); - if (0 != active) { + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS != rc) { fprintf(stderr, "Client ns %s rank %d: PMIx_Process_monitor_nb failed: %d\n", myproc.nspace, myproc.rank, rc); - exit(active); + goto done; } /* send a heartbeat */ diff --git a/opal/mca/pmix/pmix3x/pmix/examples/tool.c b/opal/mca/pmix/pmix3x/pmix/examples/tool.c index 6818e49c5a..e10699309e 100644 --- a/opal/mca/pmix/pmix3x/pmix/examples/tool.c +++ b/opal/mca/pmix/pmix3x/pmix/examples/tool.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -29,6 +29,7 @@ #include #include +#include "examples.h" static void cbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, @@ -36,15 +37,28 @@ static void cbfunc(pmix_status_t status, pmix_release_cbfunc_t release_fn, void *release_cbdata) { - volatile bool *active = (volatile bool*)cbdata; + myquery_data_t *mq = (myquery_data_t*)cbdata; + size_t n; - /* do something with the returned info - it will be + mq->lock.status = status; + + /* save the returned info - it will be * released in the release_fn */ + if (0 < ninfo) { + PMIX_INFO_CREATE(mq->info, ninfo); + mq->ninfo = ninfo; + for (n=0; n < ninfo; n++) { + PMIX_INFO_XFER(&mq->info[n], &info[n]); + } + } + /* let the library release the data */ if (NULL != release_fn) { release_fn(release_cbdata); } - *active = false; + + /* release the block */ + DEBUG_WAKEUP_THREAD(&mq->lock); } int main(int argc, char **argv) @@ -52,31 +66,154 @@ int main(int argc, char **argv) pmix_status_t rc; pmix_proc_t myproc; pmix_query_t *query; - size_t nq; - volatile bool active; + size_t nq, ninfo = 0, n, m; + myquery_data_t mydata; + pmix_info_t *info = NULL, *iptr; + char *server_uri = NULL; + char *nspace = NULL; + char *nodename = NULL; + pmix_data_array_t *darray, *dptr; + bool geturi = false; + char hostname[1024]; + + gethostname(hostname, 1024); + for (n=1; n < (size_t)argc; n++) { + if (0 == strcmp("-u", argv[n]) || 0 == strcmp("--url", argv[n])) { + if (NULL == argv[n+1]) { + fprintf(stderr, "Must provide URI argument to %s option\n", argv[n]); + exit(1); + } + server_uri = argv[n+1]; + } else if (0 == strcmp("-nspace", argv[n]) || 0 == strcmp("--nspace", argv[n])) { + if (NULL == argv[n+1]) { + fprintf(stderr, "Must provide nspace argument to %s option\n", argv[n]); + exit(1); + } + nspace = argv[n+1]; + } else if (0 == strcmp("-uri", argv[n]) || 0 == strcmp("--uri", argv[n])) { + /* retrieve the PMIx server's uri from the indicated node */ + nodename = argv[n+1]; + geturi = true; + } + } + + if (NULL != server_uri) { + ninfo = 1; + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], PMIX_SERVER_URI, server_uri, PMIX_STRING); + fprintf(stderr, "Connecting to %s\n", server_uri); + } /* init us */ - if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, NULL, 0))) { + if (PMIX_SUCCESS != (rc = PMIx_tool_init(&myproc, info, ninfo))) { fprintf(stderr, "PMIx_tool_init failed: %d\n", rc); exit(rc); } + if (NULL != info) { + PMIX_INFO_FREE(info, ninfo); + } - /* query something */ - nq = 2; - PMIX_QUERY_CREATE(query, nq); - query[0].keys = (char**)malloc(2 * sizeof(char*)); - query[0].keys[0] = strdup("foobar"); - query[0].keys[1] = NULL; - query[1].keys = (char**)malloc(2 * sizeof(char*)); - query[1].keys[0] = strdup("spastic"); - query[1].keys[1] = NULL; - active = true; - if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&active))) { - fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info failed: %d\n", myproc.nspace, myproc.rank, rc); + if (geturi) { + nq = 1; + PMIX_QUERY_CREATE(query, nq); + PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_SERVER_URI); + if (NULL != nodename) { + PMIX_QUERY_QUALIFIERS_CREATE(&query[0], 1); + PMIX_INFO_LOAD(&query[0].qualifiers[0], PMIX_HOSTNAME, nodename, PMIX_STRING); + } + DEBUG_CONSTRUCT_MYQUERY(&mydata); + if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&mydata))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info failed: %d\n", myproc.nspace, myproc.rank, rc); + goto done; + } + DEBUG_WAIT_THREAD(&mydata.lock); + /* find the response */ + if (PMIX_SUCCESS == mydata.lock.status) { + /* should be in the first key */ + if (PMIX_CHECK_KEY(&mydata.info[0], PMIX_SERVER_URI)) { + fprintf(stderr, "PMIx server URI for node %s: %s\n", + (NULL == nodename) ? hostname : nodename, + mydata.info[0].value.data.string); + } else { + fprintf(stderr, "Query returned wrong info key at first posn: %s\n", mydata.info[0].key); + } + } else { + fprintf(stderr, "Query returned error: %s\n", PMIx_Error_string(mydata.lock.status)); + } + DEBUG_DESTRUCT_MYQUERY(&mydata); goto done; } - while(active) { - usleep(10); + + if (NULL == nspace) { + /* query the list of active nspaces */ + nq = 1; + PMIX_QUERY_CREATE(query, nq); + PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_QUERY_NAMESPACE_INFO); + DEBUG_CONSTRUCT_MYQUERY(&mydata); + if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&mydata))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info failed: %d\n", myproc.nspace, myproc.rank, rc); + goto done; + } + DEBUG_WAIT_THREAD(&mydata.lock); + /* find the response */ + if (PMIX_SUCCESS == mydata.lock.status) { + /* should be in the first key */ + if (PMIX_CHECK_KEY(&mydata.info[0], PMIX_QUERY_NAMESPACE_INFO)) { + darray = mydata.info[0].value.data.darray; + fprintf(stderr, "ACTIVE NSPACES:\n"); + if (NULL == darray || 0 == darray->size || NULL == darray->array) { + fprintf(stderr, "\tNone\n"); + } else { + info = (pmix_info_t*)darray->array; + if (NULL == info) { + fprintf(stderr, "Error\n"); + } else { + for (n=0; n < darray->size; n++) { + dptr = info[n].value.data.darray; + if (NULL == dptr || 0 == dptr->size || NULL == dptr->array) { + fprintf(stderr, "Error in array %s\n", (NULL == dptr) ? "NULL" : "NON-NULL"); + break; + } + iptr = (pmix_info_t*)dptr->array; + for (m=0; m < dptr->size; m++) { + fprintf(stderr, "\t%s", iptr[m].value.data.string); + } + fprintf(stderr, "\n"); + } + } + } + } else { + fprintf(stderr, "Query returned wrong info key at first posn: %s\n", mydata.info[0].key); + } + } else { + fprintf(stderr, "Query returned error: %s\n", PMIx_Error_string(mydata.lock.status)); + } + DEBUG_DESTRUCT_MYQUERY(&mydata); + } else { + nq = 1; + PMIX_QUERY_CREATE(query, nq); + PMIX_ARGV_APPEND(rc, query[0].keys, PMIX_JOB_SIZE); + PMIX_INFO_CREATE(query[0].qualifiers, 1); + query[0].nqual = 1; + PMIX_INFO_LOAD(&query[0].qualifiers[0], PMIX_NSPACE, nspace, PMIX_STRING); + DEBUG_CONSTRUCT_MYQUERY(&mydata); + if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&mydata))) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Query_info failed: %d\n", myproc.nspace, myproc.rank, rc); + goto done; + } + DEBUG_WAIT_THREAD(&mydata.lock); + /* find the response */ + if (PMIX_SUCCESS == mydata.lock.status) { + /* should be in the first key */ + if (PMIX_CHECK_KEY(&mydata.info[0], PMIX_JOB_SIZE)) { + fprintf(stderr, "JOB SIZE FOR NSPACE %s: %lu\n", nspace, (unsigned long)mydata.info[0].value.data.uint32); + } else { + fprintf(stderr, "Query returned wrong info key at first posn: %s\n", mydata.info[0].key); + } + } else { + fprintf(stderr, "Query returned error: %s\n", PMIx_Error_string(mydata.lock.status)); + } + DEBUG_DESTRUCT_MYQUERY(&mydata); } done: diff --git a/opal/mca/pmix/pmix3x/pmix/include/Makefile.am b/opal/mca/pmix/pmix3x/pmix/include/Makefile.am index 3b205f72c5..5c0ff27f10 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/include/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2015-2018 Intel, Inc. All rights reserved. # # $COPYRIGHT$ # @@ -14,7 +14,8 @@ if WANT_PRIMARY_HEADERS include_HEADERS = \ pmix.h \ pmix_server.h \ - pmix_tool.h + pmix_tool.h \ + pmix_extend.h if WANT_PMI_BACKWARD include_HEADERS += \ diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix.h b/opal/mca/pmix/pmix3x/pmix/include/pmix.h index ae54d85b70..ba29692c99 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/pmix.h +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix.h @@ -129,7 +129,7 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int status, const char msg[], * the information locally until _PMIx_Commit_ is called. The provided scope * value is passed to the local PMIx server, which will distribute the data * as directed. */ -PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, const char key[], pmix_value_t *val); +PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, const pmix_key_t key, pmix_value_t *val); /* Push all previously _PMIx_Put_ values to the local PMIx server. @@ -200,7 +200,7 @@ PMIX_EXPORT pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs * an error. The timeout parameter can help avoid "hangs" due to programming * errors that prevent the target proc from ever exposing its data. */ -PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], +PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const pmix_key_t key, const pmix_info_t info[], size_t ninfo, pmix_value_t **val); @@ -208,7 +208,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], * be executed once the specified data has been _PMIx_Put_ * by the identified process and retrieved by the local server. The info * array is used as described above for the blocking form of this call. */ -PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char key[], +PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const pmix_key_t key, const pmix_info_t info[], size_t ninfo, pmix_value_cbfunc_t cbfunc, void *cbdata); @@ -337,7 +337,7 @@ PMIX_EXPORT pmix_status_t PMIx_Unpublish_nb(char **keys, */ PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, const pmix_app_t apps[], size_t napps, - char nspace[]); + pmix_nspace_t nspace); /* Non-blocking form of the _PMIx_Spawn_ function. The callback @@ -394,7 +394,7 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t ranges[], size_t * for releasing the array when done with it - the PMIX_PROC_FREE macro is * provided for this purpose. */ -PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, const char *nspace, +PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, const pmix_nspace_t nspace, pmix_proc_t **procs, size_t *nprocs); @@ -402,7 +402,7 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, const char *n * that nspace. The returned string will contain a comma-delimited list * of nodenames. The caller is responsible for releasing the string * when done with it */ -PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist); +PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const pmix_nspace_t nspace, char **nodelist); /* Query information about the system in general - can include * a list of active nspaces, network topology, etc. Also can be diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in b/opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in index 9554745ffc..35aecfabbc 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix_common.h.in @@ -1,10 +1,10 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. - * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -84,6 +84,10 @@ extern "C" { #define PMIX_MAX_NSLEN 255 #define PMIX_MAX_KEYLEN 511 +/* define abstract types for namespaces and keys */ +typedef char pmix_nspace_t[PMIX_MAX_NSLEN+1]; +typedef char pmix_key_t[PMIX_MAX_KEYLEN+1]; + /* define a type for rank values */ typedef uint32_t pmix_rank_t; @@ -104,6 +108,9 @@ typedef uint32_t pmix_rank_t; #define PMIX_RANK_LOCAL_NODE UINT32_MAX-2 // all ranks on local node /* define an invalid value */ #define PMIX_RANK_INVALID UINT32_MAX-3 +/* define a boundary for valid ranks */ +#define PMIX_RANK_VALID UINT32_MAX-50 + /**** PMIX ENVIRONMENTAL PARAMETERS ****/ /* There are a few environmental parameters used by PMIx for @@ -115,6 +122,9 @@ typedef uint32_t pmix_rank_t; * a tool can connect with further instructions. This envar will be * set by the tool and is _not_ intended for the direct use of users. * + * PMIX_LAUNCHER_RENDEZVOUS_FILE - if set, contains the full pathname + * of a file the launcher is to write that contains its connection info. + * Works in addition to anything else the launcher may output. */ /* define a set of "standard" PMIx attributes that can @@ -140,7 +150,6 @@ typedef uint32_t pmix_rank_t; // client rendezvous points and contact info #define PMIX_SYSTEM_TMPDIR "pmix.sys.tmpdir" // (char*) temp directory for this system, where PMIx // server will place tool rendezvous points and contact info -#define PMIX_REGISTER_NODATA "pmix.reg.nodata" // (bool) Registration is for nspace only, do not copy job data #define PMIX_SERVER_ENABLE_MONITORING "pmix.srv.monitor" // (bool) Enable PMIx internal monitoring by server #define PMIX_SERVER_NSPACE "pmix.srv.nspace" // (char*) Name of the nspace to use for this server #define PMIX_SERVER_RANK "pmix.srv.rank" // (pmix_rank_t) Rank of this server @@ -164,6 +173,7 @@ typedef uint32_t pmix_rank_t; // from the specified processes to this tool #define PMIX_RECONNECT_SERVER "pmix.cnct.recon" // (bool) tool is requesting to change server connections #define PMIX_LAUNCHER "pmix.tool.launcher" // (bool) tool is a launcher and needs rendezvous files created +#define PMIX_LAUNCHER_RENDEZVOUS_FILE "pmix.tool.lncrnd" // (char*) Pathname of file where connection info is to be stored /* identification attributes */ #define PMIX_USERID "pmix.euid" // (uint32_t) effective user id @@ -274,7 +284,6 @@ typedef uint32_t pmix_rank_t; /* topology info */ #define PMIX_NET_TOPO "pmix.ntopo" // (char*) xml-representation of network topology #define PMIX_LOCAL_TOPO "pmix.ltopo" // (char*) xml-representation of local node topology -#define PMIX_NODE_LIST "pmix.nlist" // (char*) comma-delimited list of nodes running procs for this job #define PMIX_TOPOLOGY "pmix.topo" // (hwloc_topology_t) pointer to the PMIx client's internal topology object #define PMIX_TOPOLOGY_XML "pmix.topo.xml" // (char*) XML-based description of topology #define PMIX_TOPOLOGY_FILE "pmix.topo.file" // (char*) full path to file containing XML topology description @@ -330,8 +339,6 @@ typedef uint32_t pmix_rank_t; /* event handler registration and notification info keys */ #define PMIX_EVENT_HDLR_NAME "pmix.evname" // (char*) string name identifying this handler -#define PMIX_EVENT_JOB_LEVEL "pmix.evjob" // (bool) register for job-specific events only -#define PMIX_EVENT_ENVIRO_LEVEL "pmix.evenv" // (bool) register for environment events only #define PMIX_EVENT_HDLR_FIRST "pmix.evfirst" // (bool) invoke this event handler before any other handlers #define PMIX_EVENT_HDLR_LAST "pmix.evlast" // (bool) invoke this event handler after all other handlers have been called #define PMIX_EVENT_HDLR_FIRST_IN_CATEGORY "pmix.evfirstcat" // (bool) invoke this event handler before any other handlers in this category @@ -349,7 +356,9 @@ typedef uint32_t pmix_rank_t; // registered it #define PMIX_EVENT_DO_NOT_CACHE "pmix.evnocache" // (bool) instruct the PMIx server not to cache the event #define PMIX_EVENT_SILENT_TERMINATION "pmix.evsilentterm" // (bool) do not generate an event when this job normally terminates - +#define PMIX_EVENT_PROXY "pmix.evproxy" // (pmix_proc_t*) PMIx server that sourced the event +#define PMIX_EVENT_TEXT_MESSAGE "pmix.evtext" // (char*) text message suitable for output by recipient - e.g., describing + // the cause of the event /* fault tolerance-related events */ #define PMIX_EVENT_TERMINATE_SESSION "pmix.evterm.sess" // (bool) RM intends to terminate session @@ -406,22 +415,16 @@ typedef uint32_t pmix_rank_t; #define PMIX_FWD_STDERR "pmix.fwd.stderr" // (bool) forward stderr from the spawned processes to this process (typically used by a tool) #define PMIX_FWD_STDDIAG "pmix.fwd.stddiag" // (bool) if a diagnostic channel exists, forward any output on it // from the spawned processes to this process (typically used by a tool) - - -/* connect attributes */ -#define PMIX_CONNECT_NOTIFY_EACH "pmix.cnct.each" // (bool) notify the other participants of the connection by event - // each time a process connects -#define PMIX_CONNECT_NOTIFY_REQ "pmix.cnct.req" // (bool) notify all other participants that they are requested to - // connect -#define PMIX_CONNECT_OPTIONAL "pmix.cnt.opt" // (bool) participation is optional - do not return error if procs - // terminate without having connected -#define PMIX_CONNECT_XCHG_ONLY "pmix.cnt.xchg" // (bool) provide participants with job-level info for all participating - // nspaces, but do not assign a new nspace or rank -#define PMIX_CONNECT_ID "pmix.cnt.id" // (char*) an application-provided string identifier for a PMIx_Connect operation. - +#define PMIX_SPAWN_TOOL "pmix.spwn.tool" // (bool) job being spawned is a tool +#define PMIX_CMD_LINE "pmix.cmd.line" // (char*) command line executing in the specified nspace /* query attributes */ -#define PMIX_QUERY_NAMESPACES "pmix.qry.ns" // (char*) request a comma-delimited list of active nspaces +#define PMIX_QUERY_REFRESH_CACHE "pmix.qry.rfsh" // (bool) retrieve updated information from server + // to update local cache +#define PMIX_QUERY_NAMESPACES "pmix.qry.ns" // (char*) return a comma-delimited list of active namespaces +#define PMIX_QUERY_NAMESPACE_INFO "pmix.qry.nsinfo" // (pmix_data_array_t) request an array of active nspace information - each + // element will contain an array including the namespace plus the + // command line of the application executing within it #define PMIX_QUERY_JOB_STATUS "pmix.qry.jst" // (pmix_status_t) status of a specified currently executing job #define PMIX_QUERY_QUEUE_LIST "pmix.qry.qlst" // (char*) request a comma-delimited list of scheduler queues #define PMIX_QUERY_QUEUE_STATUS "pmix.qry.qst" // (TBD) status of a specified scheduler queue @@ -442,6 +445,49 @@ typedef uint32_t pmix_rank_t; #define PMIX_TIME_REMAINING "pmix.time.remaining" // (char*) query number of seconds (uint32_t) remaining in allocation // for the specified nspace +/* information retrieval attributes */ +#define PMIX_SESSION_INFO "pmix.ssn.info" // (bool) Return information about the specified session. If information + // about a session other than the one containing the requesting + // process is desired, then the attribute array must contain a + // PMIX_SESSION_ID attribute identifying the desired target. +#define PMIX_JOB_INFO "pmix.job.info" // (bool) Return information about the specified job or namespace. If + // information about a job or namespace other than the one containing + // the requesting process is desired, then the attribute array must + // contain a PMIX_JOBID or PMIX_NSPACE attribute identifying the + // desired target. Similarly, if information is requested about a + // job or namespace in a session other than the one containing the + // requesting process, then an attribute identifying the target + // session must be provided. +#define PMIX_APP_INFO "pmix.app.info" // (bool) Return information about the specified application. If information + // about an application other than the one containing the requesting + // process is desired, then the attribute array must contain a + // PMIX_APPNUM attribute identifying the desired target. Similarly, + // if information is requested about an application in a job or session + // other than the one containing the requesting process, then attributes + // identifying the target job and/or session must be provided. +#define PMIX_NODE_INFO "pmix.node.info" // (bool) Return information about the specified node. If information about a + // node other than the one containing the requesting process is desired, + // then the attribute array must contain either the PMIX_NODEID or + // PMIX_HOSTNAME attribute identifying the desired target. + +/* information storage attributes */ +#define PMIX_SESSION_INFO_ARRAY "pmix.ssn.arr" // (pmix_data_array_t) Provide an array of pmix_info_t containing + // session-level information. The PMIX_SESSION_ID attribute is required + // to be included in the array. +#define PMIX_JOB_INFO_ARRAY "pmix.job.arr" // (pmix_data_array_t) Provide an array of pmix_info_t containing job-level + // information. Information is registered one job (aka namespace) at a time + // via the PMIx_server_register_nspace API. Thus, there is no requirement that + // the array contain either the PMIX_NSPACE or PMIX_JOBID attributes, though + // either or both of them may be included. +#define PMIX_APP_INFO_ARRAY "pmix.app.arr" // (pmix_data_array_t) Provide an array of pmix_info_t containing app-level + // information. The PMIX_NSPACE or PMIX_JOBID attributes of the job containing + // the appplication, plus its PMIX_APPNUM attribute, are required to be + // included in the array. +#define PMIX_NODE_INFO_ARRAY "pmix.node.arr" // (pmix_data_array_t) Provide an array of pmix_info_t containing node-level + // information. At a minimum, either the PMIX_NODEID or PMIX_HOSTNAME + // attribute is required to be included in the array, though both may be + // included. + /* log attributes */ #define PMIX_LOG_SOURCE "pmix.log.source" // (pmix_proc_t*) ID of source of the log request #define PMIX_LOG_STDERR "pmix.log.stderr" // (char*) log string to stderr @@ -747,75 +793,67 @@ typedef int pmix_status_t; #define PMIX_ERR_V2X_BASE -100 /* v2.x communication errors */ -#define PMIX_ERR_LOST_CONNECTION_TO_SERVER (PMIX_ERR_V2X_BASE - 1) -#define PMIX_ERR_LOST_PEER_CONNECTION (PMIX_ERR_V2X_BASE - 2) -#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT (PMIX_ERR_V2X_BASE - 3) +#define PMIX_ERR_LOST_CONNECTION_TO_SERVER -101 +#define PMIX_ERR_LOST_PEER_CONNECTION -102 +#define PMIX_ERR_LOST_CONNECTION_TO_CLIENT -103 /* used by the query system */ -#define PMIX_QUERY_PARTIAL_SUCCESS (PMIX_ERR_V2X_BASE - 4) +#define PMIX_QUERY_PARTIAL_SUCCESS -104 /* request responses */ -#define PMIX_NOTIFY_ALLOC_COMPLETE (PMIX_ERR_V2X_BASE - 5) +#define PMIX_NOTIFY_ALLOC_COMPLETE -105 /* job control */ -#define PMIX_JCTRL_CHECKPOINT (PMIX_ERR_V2X_BASE - 6) // monitored by client to trigger checkpoint operation -#define PMIX_JCTRL_CHECKPOINT_COMPLETE (PMIX_ERR_V2X_BASE - 7) // sent by client and monitored by server to notify that requested +#define PMIX_JCTRL_CHECKPOINT -106 // monitored by client to trigger checkpoint operation +#define PMIX_JCTRL_CHECKPOINT_COMPLETE -107 // sent by client and monitored by server to notify that requested // checkpoint operation has completed -#define PMIX_JCTRL_PREEMPT_ALERT (PMIX_ERR_V2X_BASE - 8) // monitored by client to detect RM intends to preempt -/* monitoring */ -#define PMIX_MONITOR_HEARTBEAT_ALERT (PMIX_ERR_V2X_BASE - 9) -#define PMIX_MONITOR_FILE_ALERT (PMIX_ERR_V2X_BASE - 10) -#define PMIX_PROC_TERMINATED (PMIX_ERR_V2X_BASE - 11) +#define PMIX_JCTRL_PREEMPT_ALERT -108 // monitored by client to detect RM intends to preempt -/* define a starting point for operational error constants so - * we avoid renumbering when making additions */ -#define PMIX_ERR_OP_BASE PMIX_ERR_V2X_BASE-30 +/* monitoring */ +#define PMIX_MONITOR_HEARTBEAT_ALERT -109 +#define PMIX_MONITOR_FILE_ALERT -110 +#define PMIX_PROC_TERMINATED -111 +#define PMIX_ERR_INVALID_TERMINATION -112 /* operational */ -#define PMIX_ERR_EVENT_REGISTRATION (PMIX_ERR_OP_BASE - 14) -#define PMIX_ERR_JOB_TERMINATED (PMIX_ERR_OP_BASE - 15) -#define PMIX_ERR_UPDATE_ENDPOINTS (PMIX_ERR_OP_BASE - 16) -#define PMIX_MODEL_DECLARED (PMIX_ERR_OP_BASE - 17) -#define PMIX_GDS_ACTION_COMPLETE (PMIX_ERR_OP_BASE - 18) -#define PMIX_PROC_HAS_CONNECTED (PMIX_ERR_OP_BASE - 19) -#define PMIX_CONNECT_REQUESTED (PMIX_ERR_OP_BASE - 20) -#define PMIX_MODEL_RESOURCES (PMIX_ERR_OP_BASE - 21) // model resource usage has changed -#define PMIX_OPENMP_PARALLEL_ENTERED (PMIX_ERR_OP_BASE - 22) // an OpenMP parallel region has been entered -#define PMIX_OPENMP_PARALLEL_EXITED (PMIX_ERR_OP_BASE - 23) // an OpenMP parallel region has completed -#define PMIX_LAUNCH_DIRECTIVE (PMIX_ERR_OP_BASE - 24) -#define PMIX_LAUNCHER_READY (PMIX_ERR_OP_BASE - 25) -#define PMIX_OPERATION_IN_PROGRESS (PMIX_ERR_OP_BASE - 26) -#define PMIX_OPERATION_SUCCEEDED (PMIX_ERR_OP_BASE - 27) -/* gap for group codes */ - - -/* define a starting point for system error constants so - * we avoid renumbering when making additions */ -#define PMIX_ERR_SYS_BASE PMIX_ERR_OP_BASE-100 +#define PMIX_ERR_EVENT_REGISTRATION -144 +#define PMIX_ERR_JOB_TERMINATED -145 +#define PMIX_ERR_UPDATE_ENDPOINTS -146 +#define PMIX_MODEL_DECLARED -147 +#define PMIX_GDS_ACTION_COMPLETE -148 +#define PMIX_PROC_HAS_CONNECTED -149 +#define PMIX_CONNECT_REQUESTED -150 +#define PMIX_MODEL_RESOURCES -151 // model resource usage has changed +#define PMIX_OPENMP_PARALLEL_ENTERED -152 // an OpenMP parallel region has been entered +#define PMIX_OPENMP_PARALLEL_EXITED -153 // an OpenMP parallel region has completed +#define PMIX_LAUNCH_DIRECTIVE -154 +#define PMIX_LAUNCHER_READY -155 +#define PMIX_OPERATION_IN_PROGRESS -156 +#define PMIX_OPERATION_SUCCEEDED -157 +#define PMIX_ERR_INVALID_OPERATION -158 /* system failures */ -#define PMIX_ERR_NODE_DOWN (PMIX_ERR_SYS_BASE - 1) -#define PMIX_ERR_NODE_OFFLINE (PMIX_ERR_SYS_BASE - 2) +#define PMIX_ERR_NODE_DOWN -231 +#define PMIX_ERR_NODE_OFFLINE -232 +#define PMIX_ERR_SYS_OTHER -330 - -/* define a starting point for event handler error constants so - * we avoid renumbering when making additions */ -#define PMIX_ERR_EVHDLR_BASE PMIX_ERR_SYS_BASE-100 +/* define a macro for identifying system event values */ +#define PMIX_SYSTEM_EVENT(a) \ + ((a) <= PMIX_ERR_NODE_DOWN && PMIX_ERR_SYS_OTHER <= (a)) /* used by event handlers */ -#define PMIX_EVENT_NO_ACTION_TAKEN (PMIX_ERR_EVHDLR_BASE - 1) -#define PMIX_EVENT_PARTIAL_ACTION_TAKEN (PMIX_ERR_EVHDLR_BASE - 2) -#define PMIX_EVENT_ACTION_DEFERRED (PMIX_ERR_EVHDLR_BASE - 3) -#define PMIX_EVENT_ACTION_COMPLETE (PMIX_ERR_EVHDLR_BASE - 4) - +#define PMIX_EVENT_NO_ACTION_TAKEN -331 +#define PMIX_EVENT_PARTIAL_ACTION_TAKEN -332 +#define PMIX_EVENT_ACTION_DEFERRED -333 +#define PMIX_EVENT_ACTION_COMPLETE -334 /* define a starting point for PMIx internal error codes * that are never exposed outside the library */ -#define PMIX_INTERNAL_ERR_BASE -1000 +#define PMIX_INTERNAL_ERR_BASE -1330 /* define a starting point for user-level defined error * constants - negative values larger than this are guaranteed * not to conflict with PMIx values. Definitions should always * be based on the PMIX_EXTERNAL_ERR_BASE constant and -not- a * specific value as the value of the constant may change */ -#define PMIX_EXTERNAL_ERR_BASE -2000 +#define PMIX_EXTERNAL_ERR_BASE PMIX_INTERNAL_ERR_BASE-2000 /**** PMIX DATA TYPES ****/ typedef uint16_t pmix_data_type_t; @@ -850,7 +888,7 @@ typedef uint16_t pmix_data_type_t; #define PMIX_BUFFER 26 #define PMIX_BYTE_OBJECT 27 #define PMIX_KVAL 28 -#define PMIX_MODEX 29 +// Hole left by deprecation/removal of PMIX_MODEX #define PMIX_PERSIST 30 #define PMIX_POINTER 31 #define PMIX_SCOPE 32 @@ -865,9 +903,7 @@ typedef uint16_t pmix_data_type_t; #define PMIX_QUERY 41 #define PMIX_COMPRESSED_STRING 42 // string compressed with zlib #define PMIX_ALLOC_DIRECTIVE 43 -/**** DEPRECATED ****/ -#define PMIX_INFO_ARRAY 44 -/**** ****/ +// Hole left by deprecation/removal of PMIX_INFO_ARRAY #define PMIX_IOF_CHANNEL 45 #define PMIX_ENVAR 46 /********************/ @@ -894,7 +930,7 @@ typedef uint8_t pmix_scope_t; #define PMIX_GLOBAL 3 // share with all procs (local + remote) #define PMIX_INTERNAL 4 // store data in the internal tables -/* define a range for data "published" by PMI +/* define a range for data "published" by PMIx */ typedef uint8_t pmix_data_range_t; #define PMIX_RANGE_UNDEF 0 @@ -920,6 +956,7 @@ typedef uint8_t pmix_persistence_t; * command directives via pmix_info_t arrays */ typedef uint32_t pmix_info_directives_t; #define PMIX_INFO_REQD 0x00000001 +#define PMIX_INFO_ARRAY_END 0x00000002 // mark the end of an array created by PMIX_INFO_CREATE /* the top 16-bits are reserved for internal use by * implementers - these may be changed inside the * PMIx library */ @@ -952,15 +989,34 @@ typedef uint16_t pmix_iof_channel_t; #define PMIX_FWD_STDDIAG_CHANNEL 0x0008 #define PMIX_FWD_ALL_CHANNELS 0x00ff - /* declare a convenience macro for checking keys */ #define PMIX_CHECK_KEY(a, b) \ (0 == strncmp((a)->key, (b), PMIX_MAX_KEYLEN)) +#define PMIX_LOAD_KEY(a, b) \ + do { \ + memset((a), 0, PMIX_MAX_KEYLEN+1); \ + pmix_strncpy((a), (b), PMIX_MAX_KEYLEN); \ + }while(0) + +/* define a convenience macro for loading nspaces */ +#define PMIX_LOAD_NSPACE(a, b) \ + do { \ + memset((a), 0, PMIX_MAX_NSLEN+1); \ + pmix_strncpy((a), (b), PMIX_MAX_NSLEN); \ + }while(0) + /* define a convenience macro for checking nspaces */ #define PMIX_CHECK_NSPACE(a, b) \ (0 == strncmp((a), (b), PMIX_MAX_NSLEN)) +/* define a convenience macro for loading names */ +#define PMIX_LOAD_PROCID(a, b, c) \ + do { \ + PMIX_LOAD_NSPACE((a)->nspace, (b)); \ + (a)->rank = (c); \ + }while(0) + /* define a convenience macro for checking names */ #define PMIX_CHECK_PROCID(a, b) \ (PMIX_CHECK_NSPACE((a)->nspace, (b)->nspace) && ((a)->rank == (b)->rank || (PMIX_RANK_WILDCARD == (a)->rank || PMIX_RANK_WILDCARD == (b)->rank))) @@ -993,15 +1049,18 @@ typedef struct pmix_byte_object { } \ } while(0) -#define PMIX_BYTE_OBJECT_FREE(m, n) \ - do { \ - size_t _n; \ - for (_n=0; _n < n; _n++) { \ - if (NULL != (m)[_n].bytes) { \ - free((m)[_n].bytes); \ - } \ - } \ - free((m)); \ +#define PMIX_BYTE_OBJECT_FREE(m, n) \ + do { \ + size_t _bon; \ + if (NULL != (m)) { \ + for (_bon=0; _bon < n; _bon++) { \ + if (NULL != (m)[_bon].bytes) { \ + free((m)[_bon].bytes); \ + } \ + } \ + free((m)); \ + (m) = NULL; \ + } \ } while(0) #define PMIX_BYTE_OBJECT_LOAD(b, d, s) \ @@ -1034,10 +1093,10 @@ typedef struct { } while (0) #define PMIX_ENVAR_FREE(m, n) \ do { \ - size_t _k; \ + size_t _ek; \ if (NULL != (m)) { \ - for (_k=0; _k < (n); _k++) { \ - PMIX_ENVAR_DESTRUCT(&(m)[_k]); \ + for (_ek=0; _ek < (n); _ek++) { \ + PMIX_ENVAR_DESTRUCT(&(m)[_ek]); \ } \ free((m)); \ } \ @@ -1130,7 +1189,7 @@ typedef struct pmix_data_buffer { /**** PMIX PROC OBJECT ****/ typedef struct pmix_proc { - char nspace[PMIX_MAX_NSLEN+1]; + pmix_nspace_t nspace; pmix_rank_t rank; } pmix_proc_t; #define PMIX_PROC_CREATE(m, n) \ @@ -1162,7 +1221,7 @@ typedef struct pmix_proc { #define PMIX_PROC_LOAD(m, n, r) \ do { \ PMIX_PROC_CONSTRUCT((m)); \ - (void)strncpy((m)->nspace, (n), PMIX_MAX_NSLEN); \ + pmix_strncpy((m)->nspace, (n), PMIX_MAX_NSLEN); \ (m)->rank = (r); \ } while(0) @@ -1172,9 +1231,9 @@ typedef struct pmix_proc { memset((t), 0, PMIX_MAX_NSLEN+1); \ _len = strlen((c)); \ if ((_len + strlen((n))) < PMIX_MAX_NSLEN) { \ - (void)strncpy((t), (c), PMIX_MAX_NSLEN); \ + pmix_strncpy((t), (c), PMIX_MAX_NSLEN); \ (t)[_len] = ':'; \ - (void)strncpy(&(t)[_len+1], (n), PMIX_MAX_NSLEN - _len - 1); \ + pmix_strncpy(&(t)[_len+1], (n), PMIX_MAX_NSLEN - _len); \ } \ } while(0) @@ -1241,34 +1300,24 @@ typedef struct pmix_proc_info { } while (0) -/**** PMIX VALUE STRUCT ****/ -typedef struct pmix_info_t pmix_info_t; +/**** PMIX DATA ARRAY STRUCT ****/ typedef struct pmix_data_array { pmix_data_type_t type; size_t size; void *array; } pmix_data_array_t; -#define PMIX_DATA_ARRAY_CONSTRUCT(m, n, t) \ - do { \ - (m)->type = (t); \ - (m)->size = (n); \ - } while(0) -#define PMIX_DATA_ARRAY_CREATE(m, n, t) \ - do { \ - (m) = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); \ - PMIX_DATA_ARRAY_CONSTRUCT((m), (n), (t)); \ - } while(0) -typedef struct pmix_info_array { - size_t size; - pmix_info_t *array; -} pmix_info_array_t; -/********************/ +/**** THE PMIX_DATA_ARRAY SUPPORT MACROS ARE DEFINED ****/ +/**** DOWN BELOW (NEAR THE BOTTOM OF THE FILE) TO ****/ +/**** AVOID CIRCULAR DEPENDENCIES ****/ + + +/**** PMIX VALUE STRUCT ****/ /* NOTE: operations can supply a collection of values under - * a single key by passing a pmix_value_t containing an - * array of type PMIX_INFO_ARRAY, with each array element + * a single key by passing a pmix_value_t containing a + * data array of type PMIX_INFO, with each array element * containing its own pmix_info_t object */ typedef struct pmix_value { @@ -1306,9 +1355,6 @@ typedef struct pmix_value { void *ptr; pmix_alloc_directive_t adir; pmix_envar_t envar; - /**** DEPRECATED ****/ - pmix_info_array_t *array; - /********************/ } data; } pmix_value_t; /* allocate and initialize a specified number of value structs */ @@ -1345,16 +1391,36 @@ typedef struct pmix_value { #define PMIX_VALUE_FREE(m, n) \ do { \ - size_t _s; \ + size_t _vv; \ if (NULL != (m)) { \ - for (_s=0; _s < (n); _s++) { \ - PMIX_VALUE_DESTRUCT(&((m)[_s])); \ + for (_vv=0; _vv < (n); _vv++) { \ + PMIX_VALUE_DESTRUCT(&((m)[_vv])); \ } \ free((m)); \ (m) = NULL; \ } \ } while (0) +#define PMIX_VALUE_LOAD(v, d, t) \ + pmix_value_load((v), (d), (t)) + +#define PMIX_VALUE_UNLOAD(r, k, d, s) \ + (r) = pmix_value_unload((k), (d), (s)) + +#define PMIX_VALUE_XFER(r, v, s) \ + do { \ + if (NULL == (v)) { \ + (v) = (pmix_value_t*)malloc(sizeof(pmix_value_t)); \ + if (NULL == (v)) { \ + (r) = PMIX_ERR_NOMEM; \ + } else { \ + (r) = pmix_value_xfer((v), (s)); \ + } \ + } else { \ + (r) = pmix_value_xfer((v), (s)); \ + } \ + } while(0) + #define PMIX_VALUE_GET_NUMBER(s, m, n, t) \ do { \ (s) = PMIX_SUCCESS; \ @@ -1384,60 +1450,47 @@ typedef struct pmix_value { (n) = (t)((m)->data.fval); \ } else if (PMIX_DOUBLE == (m)->type) { \ (n) = (t)((m)->data.dval); \ + } else if (PMIX_PID == (m)->type) { \ + (n) = (t)((m)->data.pid); \ } else { \ (s) = PMIX_ERR_BAD_PARAM; \ } \ } while(0) -/* expose some functions that are resolved in the - * PMIx library, but part of a header that - * includes internal functions - we don't - * want to expose the entire header here. For - * consistency, we provide macro versions as well - */ -void pmix_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); -#define PMIX_VALUE_LOAD(v, d, t) \ - pmix_value_load((v), (d), (t)) - -pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, size_t *sz); -#define PMIX_VALUE_UNLOAD(r, k, d, s) \ - (r) = pmix_value_unload((k), (d), (s)) - -pmix_status_t pmix_value_xfer(pmix_value_t *kv, pmix_value_t *src); -#define PMIX_VALUE_XFER(r, v, s) \ - do { \ - if (NULL == (v)) { \ - (v) = (pmix_value_t*)malloc(sizeof(pmix_value_t)); \ - if (NULL == (v)) { \ - (r) = PMIX_ERR_NOMEM; \ - } else { \ - (r) = pmix_value_xfer((v), (s)); \ - } \ - } else { \ - (r) = pmix_value_xfer((v), (s)); \ - } \ +#define PMIX_VALUE_COMPRESSED_STRING_UNPACK(s) \ + do { \ + char *tmp; \ + /* if this is a compressed string, then uncompress it */ \ + if (PMIX_COMPRESSED_STRING == (s)->type) { \ + pmix_util_uncompress_string(&tmp, (uint8_t*)(s)->data.bo.bytes, \ + (s)->data.bo.size); \ + if (NULL == tmp) { \ + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); \ + rc = PMIX_ERR_NOMEM; \ + PMIX_VALUE_RELEASE(s); \ + val = NULL; \ + } else { \ + PMIX_VALUE_DESTRUCT(s); \ + (s)->data.string = tmp; \ + (s)->type = PMIX_STRING; \ + } \ + } \ } while(0) -pmix_status_t pmix_argv_append_nosize(char ***argv, const char *arg); -#define PMIX_ARGV_APPEND(r, a, b) \ - (r) = pmix_argv_append_nosize(&(a), (b)) - -pmix_status_t pmix_setenv(const char *name, const char *value, - bool overwrite, char ***env); -#define PMIX_SETENV(r, a, b, c) \ - (r) = pmix_setenv((a), (b), true, (c)) - /**** PMIX INFO STRUCT ****/ -struct pmix_info_t { - char key[PMIX_MAX_KEYLEN+1]; // ensure room for the NULL terminator +typedef struct pmix_info { + pmix_key_t key; pmix_info_directives_t flags; // bit-mask of flags pmix_value_t value; -}; +} pmix_info_t; /* utility macros for working with pmix_info_t structs */ #define PMIX_INFO_CREATE(m, n) \ do { \ + pmix_info_t *_i; \ (m) = (pmix_info_t*)calloc((n), sizeof(pmix_info_t)); \ + _i = (pmix_info_t*)(m); \ + _i[(n)-1].flags = PMIX_INFO_ARRAY_END; \ } while (0) #define PMIX_INFO_CONSTRUCT(m) \ @@ -1451,22 +1504,22 @@ struct pmix_info_t { PMIX_VALUE_DESTRUCT(&(m)->value); \ } while (0) -#define PMIX_INFO_FREE(m, n) \ - do { \ - size_t _s; \ - if (NULL != (m)) { \ - for (_s=0; _s < (n); _s++) { \ - PMIX_INFO_DESTRUCT(&((m)[_s])); \ - } \ - free((m)); \ - (m) = NULL; \ - } \ +#define PMIX_INFO_FREE(m, n) \ + do { \ + size_t _is; \ + if (NULL != (m)) { \ + for (_is=0; _is < (n); _is++) { \ + PMIX_INFO_DESTRUCT(&((m)[_is])); \ + } \ + free((m)); \ + (m) = NULL; \ + } \ } while (0) #define PMIX_INFO_LOAD(m, k, v, t) \ do { \ if (NULL != (k)) { \ - (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ + pmix_strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ } \ (m)->flags = 0; \ pmix_value_load(&((m)->value), (v), (t)); \ @@ -1474,46 +1527,30 @@ struct pmix_info_t { #define PMIX_INFO_XFER(d, s) \ do { \ if (NULL != (s)->key) { \ - (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ + pmix_strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ } \ (d)->flags = (s)->flags; \ pmix_value_xfer(&(d)->value, (pmix_value_t*)&(s)->value); \ } while(0) + +/* macros for setting and unsetting the "reqd" flag + * in a pmix_info_t */ #define PMIX_INFO_REQUIRED(m) \ (m)->flags |= PMIX_INFO_REQD #define PMIX_INFO_OPTIONAL(m) \ (m)->flags &= ~PMIX_INFO_REQD +/* macros for testing the "reqd" flag in a pmix_info_t */ #define PMIX_INFO_IS_REQUIRED(m) \ (m)->flags & PMIX_INFO_REQD #define PMIX_INFO_IS_OPTIONAL(m) \ !((m)->flags & PMIX_INFO_REQD) -#define PMIX_INFO_UNLOAD(r, v, l) \ - do { \ - pmix_info_t *_info; \ - size_t _n, _ninfo; \ - pmix_kval_t *_kv; \ - _info = (pmix_info_t*)(v)->data.darray->array; \ - _ninfo = (v)->data.darray->size; \ - for (_n = 0; _n < _ninfo; _n++){ \ - _kv = PMIX_NEW(pmix_kval_t); \ - if (NULL == _kv) { \ - (r) = PMIX_ERR_NOMEM; \ - break; \ - } \ - if (NULL != _info[_n].key) { \ - _kv->key = strdup(_info[_n].key); \ - } \ - PMIX_VALUE_XFER((r), _kv->value, &_info[_n].value);\ - if (PMIX_SUCCESS != (r)) { \ - PMIX_RELEASE(_kv); \ - break; \ - } \ - pmix_list_append((l), &_kv->super); \ - } \ - } while(0) +/* macro for testing end of the array */ +#define PMIX_INFO_IS_END(m) \ + (m)->flags & PMIX_INFO_ARRAY_END + /* define a special macro for checking if a boolean * info is true - when info structs are provided, a * type of PMIX_UNDEF is taken to imply a boolean "true" @@ -1526,7 +1563,7 @@ struct pmix_info_t { /**** PMIX LOOKUP RETURN STRUCT ****/ typedef struct pmix_pdata { pmix_proc_t proc; - char key[PMIX_MAX_KEYLEN+1]; // ensure room for the NULL terminator + pmix_key_t key; pmix_value_t value; } pmix_pdata_t; @@ -1556,10 +1593,11 @@ typedef struct pmix_pdata { #define PMIX_PDATA_FREE(m, n) \ do { \ - size_t _s; \ - if (NULL != (m)) { \ - for (_s=0; _s < (n); _s++) { \ - PMIX_PDATA_DESTRUCT(&((m)[_s])); \ + size_t _ps; \ + pmix_pdata_t *_pdf = (pmix_pdata_t*)(m); \ + if (NULL != _pdf) { \ + for (_ps=0; _ps < (n); _ps++) { \ + PMIX_PDATA_DESTRUCT(&(_pdf[_ps])); \ } \ free((m)); \ (m) = NULL; \ @@ -1570,9 +1608,9 @@ typedef struct pmix_pdata { do { \ if (NULL != (m)) { \ memset((m), 0, sizeof(pmix_pdata_t)); \ - (void)strncpy((m)->proc.nspace, (p)->nspace, PMIX_MAX_NSLEN); \ + pmix_strncpy((m)->proc.nspace, (p)->nspace, PMIX_MAX_NSLEN); \ (m)->proc.rank = (p)->rank; \ - (void)strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ + pmix_strncpy((m)->key, (k), PMIX_MAX_KEYLEN); \ pmix_value_load(&((m)->value), (v), (t)); \ } \ } while (0) @@ -1581,9 +1619,9 @@ typedef struct pmix_pdata { do { \ if (NULL != (d)) { \ memset((d), 0, sizeof(pmix_pdata_t)); \ - (void)strncpy((d)->proc.nspace, (s)->proc.nspace, PMIX_MAX_NSLEN); \ + pmix_strncpy((d)->proc.nspace, (s)->proc.nspace, PMIX_MAX_NSLEN); \ (d)->proc.rank = (s)->proc.rank; \ - (void)strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ + pmix_strncpy((d)->key, (s)->key, PMIX_MAX_KEYLEN); \ pmix_value_xfer(&((d)->value), &((s)->value)); \ } \ } while (0) @@ -1605,6 +1643,12 @@ typedef struct pmix_app { (m) = (pmix_app_t*)calloc((n), sizeof(pmix_app_t)); \ } while (0) +#define PMIX_APP_INFO_CREATE(m, n) \ + do { \ + (m)->ninfo = (n); \ + PMIX_INFO_CREATE((m)->info, (m)->ninfo); \ + } while(0) + #define PMIX_APP_RELEASE(m) \ do { \ PMIX_APP_DESTRUCT((m)); \ @@ -1619,21 +1663,21 @@ typedef struct pmix_app { #define PMIX_APP_DESTRUCT(m) \ do { \ - size_t _ii; \ + size_t _aii; \ if (NULL != (m)->cmd) { \ free((m)->cmd); \ (m)->cmd = NULL; \ } \ if (NULL != (m)->argv) { \ - for (_ii=0; NULL != (m)->argv[_ii]; _ii++) { \ - free((m)->argv[_ii]); \ + for (_aii=0; NULL != (m)->argv[_aii]; _aii++) { \ + free((m)->argv[_aii]); \ } \ free((m)->argv); \ (m)->argv = NULL; \ } \ if (NULL != (m)->env) { \ - for (_ii=0; NULL != (m)->env[_ii]; _ii++) { \ - free((m)->env[_ii]); \ + for (_aii=0; NULL != (m)->env[_aii]; _aii++) { \ + free((m)->env[_aii]); \ } \ free((m)->env); \ (m)->env = NULL; \ @@ -1643,20 +1687,18 @@ typedef struct pmix_app { (m)->cwd = NULL; \ } \ if (NULL != (m)->info) { \ - for (_ii=0; _ii < (m)->ninfo; _ii++) { \ - PMIX_INFO_DESTRUCT(&(m)->info[_ii]); \ - } \ - free((m)->info); \ + PMIX_INFO_FREE((m)->info, (m)->ninfo); \ (m)->info = NULL; \ + (m)->ninfo = 0; \ } \ } while (0) #define PMIX_APP_FREE(m, n) \ do { \ - size_t _s; \ + size_t _as; \ if (NULL != (m)) { \ - for (_s=0; _s < (n); _s++) { \ - PMIX_APP_DESTRUCT(&((m)[_s])); \ + for (_as=0; _as < (n); _as++) { \ + PMIX_APP_DESTRUCT(&((m)[_as])); \ } \ free((m)); \ (m) = NULL; \ @@ -1676,6 +1718,12 @@ typedef struct pmix_query { (m) = (pmix_query_t*)calloc((n) , sizeof(pmix_query_t)); \ } while (0) +#define PMIX_QUERY_QUALIFIERS_CREATE(m, n) \ + do { \ + (m)->nqual = (n); \ + PMIX_INFO_CREATE((m)->qualifiers, (m)->nqual); \ + } while(0) + #define PMIX_QUERY_RELEASE(m) \ do { \ PMIX_QUERY_DESTRUCT((m)); \ @@ -1690,29 +1738,27 @@ typedef struct pmix_query { #define PMIX_QUERY_DESTRUCT(m) \ do { \ - size_t _ii; \ + size_t _qi; \ if (NULL != (m)->keys) { \ - for (_ii=0; NULL != (m)->keys[_ii]; _ii++) { \ - free((m)->keys[_ii]); \ + for (_qi=0; NULL != (m)->keys[_qi]; _qi++) { \ + free((m)->keys[_qi]); \ } \ free((m)->keys); \ (m)->keys = NULL; \ } \ if (NULL != (m)->qualifiers) { \ - for (_ii=0; _ii < (m)->nqual; _ii++) { \ - PMIX_INFO_DESTRUCT(&(m)->qualifiers[_ii]); \ - } \ - free((m)->qualifiers); \ + PMIX_INFO_FREE((m)->qualifiers, (m)->nqual); \ (m)->qualifiers = NULL; \ + (m)->nqual = 0; \ } \ } while (0) #define PMIX_QUERY_FREE(m, n) \ do { \ - size_t _s; \ + size_t _qs; \ if (NULL != (m)) { \ - for (_s=0; _s < (n); _s++) { \ - PMIX_QUERY_DESTRUCT(&((m)[_s])); \ + for (_qs=0; _qs < (n); _qs++) { \ + PMIX_QUERY_DESTRUCT(&((m)[_qs])); \ } \ free((m)); \ (m) = NULL; \ @@ -1720,52 +1766,152 @@ typedef struct pmix_query { } while (0) +/**** GENERIC HELPER MACROS ****/ +/* Append a string (by value) to an new or existing NULL-terminated + * argv array. + * + * @param argv Pointer to an argv array. + * @param str Pointer to the string to append. + * + * @retval PMIX_SUCCESS On success + * @retval PMIX_ERROR On failure + * + * This function adds a string to an argv array of strings by value; + * it is permissable to pass a string on the stack as the str + * argument to this function. + * + * To add the first entry to an argv array, call this function with + * (*argv == NULL). This function will allocate an array of length + * 2; the first entry will point to a copy of the string passed in + * arg, the second entry will be set to NULL. + * + * If (*argv != NULL), it will be realloc'ed to be 1 (char*) larger, + * and the next-to-last entry will point to a copy of the string + * passed in arg. The last entry will be set to NULL. + * + * Just to reinforce what was stated above: the string is copied by + * value into the argv array; there is no need to keep the original + * string (i.e., the arg parameter) after invoking this function. + */ +#define PMIX_ARGV_APPEND(r, a, b) \ + (r) = pmix_argv_append_nosize(&(a), (b)) -/**** PMIX MODEX STRUCT ****/ -typedef struct pmix_modex_data { - char nspace[PMIX_MAX_NSLEN+1]; - int rank; - uint8_t *blob; - size_t size; -} pmix_modex_data_t; -/* utility macros for working with pmix_modex_t structs */ -#define PMIX_MODEX_CREATE(m, n) \ - do { \ - (m) = (pmix_modex_data_t*)calloc((n) , sizeof(pmix_modex_data_t)); \ - } while (0) +/* Prepend a string to a new or existing NULL-terminated + * argv array - same as above only prepend + */ +#define PMIX_ARGV_PREPEND(r, a, b) \ + (r) = pmix_argv_prepend_nosize(a, b) -#define PMIX_MODEX_RELEASE(m) \ - do { \ - PMIX_MODEX_DESTRUCT((m)); \ - free((m)); \ - (m) = NULL; \ - } while (0) +/* Append to an argv-style array, but only if the provided argument + * doesn't already exist somewhere in the array. Ignore the size of the array. + * + * @param argv Pointer to an argv array. + * @param str Pointer to the string to append. + * @param bool Whether or not to overwrite a matching value if found + * + * @retval PMIX_SUCCESS On success + * @retval PMIX_ERROR On failure + * + * This function is identical to the pmix_argv_append_nosize() function + * except that it only appends the provided argument if it does not already + * exist in the provided array, or overwrites it if it is. + */ +#define PMIX_ARGV_APPEND_UNIQUE(r, a, b, c) \ + (r) = pmix_argv_append_unique_nosize(a, b, c) -#define PMIX_MODEX_CONSTRUCT(m) \ - do { \ - memset((m), 0, sizeof(pmix_modex_data_t)); \ - } while (0) +/* Free a NULL-terminated argv array. + * + * @param argv Argv array to free. + * + * This function frees an argv array and all of the strings that it + * contains. Since the argv parameter is passed by value, it is not + * set to NULL in the caller's scope upon return. + * + * It is safe to invoke this function with a NULL pointer. It is + * not safe to invoke this function with a non-NULL-terminated argv + * array. + */ +#define PMIX_ARGV_FREE(a) pmix_argv_free(a) -#define PMIX_MODEX_DESTRUCT(m) \ - do { \ - if (NULL != (m)->blob) { \ - free((m)->blob); \ - (m)->blob = NULL; \ - } \ - } while (0) +/* + * Split a string into a NULL-terminated argv array. Do not include empty + * strings in result array. + * + * @param src_string Input string. + * @param delimiter Delimiter character. + * + * @retval argv pointer to new argv array on success + * @retval NULL on error + * + * All strings are insertted into the argv array by value; the + * newly-allocated array makes no references to the src_string + * argument (i.e., it can be freed after calling this function + * without invalidating the output argv). + */ +#define PMIX_ARGV_SPLIT(a, b, c) \ + (a) = pmix_argv_split(b, c) -#define PMIX_MODEX_FREE(m, n) \ - do { \ - size_t _s; \ - if (NULL != (m)) { \ - for (_s=0; _s < (n); _s++) { \ - PMIX_MODEX_DESTRUCT(&((m)[_s])); \ - } \ - free((m)); \ - (m) = NULL; \ - } \ - } while (0) +/* + * Return the length of a NULL-terminated argv array. + * + * @param argv The input argv array. + * + * @retval 0 If NULL is passed as argv. + * @retval count Number of entries in the argv array. + * + * The argv array must be NULL-terminated. + */ +#define PMIX_ARGV_COUNT(r, a) \ + (r) = pmix_argv_count(a) + +/* + * Join all the elements of an argv array into a single + * newly-allocated string. + * + * @param argv The input argv array. + * @param delimiter Delimiter character placed between each argv string. + * + * @retval new_string Output string on success. + * @retval NULL On failure. + * + * Similar to the Perl join function, this function takes an input + * argv and joins them into into a single string separated by the + * delimiter character. + * + * It is the callers responsibility to free the returned string. + */ +#define PMIX_ARGV_JOIN(a, b, c) \ + (a) = pmix_argv_join(b, c) + +/* + * Copy a NULL-terminated argv array. + * + * @param argv The input argv array. + * + * @retval argv Copied argv array on success. + * @retval NULL On failure. + * + * Copy an argv array, including copying all off its strings. + * Specifically, the output argv will be an array of the same length + * as the input argv, and strcmp(argv_in[i], argv_out[i]) will be 0. + */ +#define PMIX_ARGV_COPY(a, b) \ + (a) = pmix_argv_copy(b) + +/* + * Set an environmenal paramter in an env array + * + * @retval r Return pmix_status_t status + * + * @param a Name of the environmental param + * + * @param b String value of the environmental param + * + * @param c Address of the NULL-terminated env array + */ +#define PMIX_SETENV(r, a, b, c) \ + (r) = pmix_setenv((a), (b), true, (c)) /**** CALLBACK FUNCTIONS FOR NON-BLOCKING OPERATIONS ****/ @@ -1794,7 +1940,7 @@ typedef void (*pmix_modex_cbfunc_t)(pmix_status_t status, * released by the library upon return from the callback function, so * the receiver must copy it if it needs to be retained */ typedef void (*pmix_spawn_cbfunc_t)(pmix_status_t status, - char nspace[], void *cbdata); + pmix_nspace_t nspace, void *cbdata); /* define a callback for common operations that simply return * a status. Examples include the non-blocking versions of @@ -1902,6 +2048,10 @@ typedef void (*pmix_notification_fn_t)(size_t evhdlr_registration_id, typedef void (*pmix_hdlr_reg_cbfunc_t)(pmix_status_t status, size_t refid, void *cbdata); +/* maintain backward compatibility with v2 definition - change of name */ +typedef void (*pmix_evhdlr_reg_cbfunc_t)(pmix_status_t status, + size_t evhdlr_ref, + void *cbdata); /* define a callback function for calls to PMIx_Get_nb. The status * indicates if the requested data was found or not - a pointer to the @@ -2080,7 +2230,7 @@ PMIX_EXPORT void PMIx_Deregister_event_handler(size_t evhdlr_ref, PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, const pmix_proc_t *source, pmix_data_range_t range, - pmix_info_t info[], size_t ninfo, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); /* Provide a string representation for several types of value. Note @@ -2114,7 +2264,7 @@ PMIX_EXPORT const char* PMIx_Get_version(void); * proc. This is data that has only internal scope - it will * never be "pushed" externally */ PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc, - const char *key, pmix_value_t *val); + const pmix_key_t key, pmix_value_t *val); /** * Top-level interface function to pack one or more values into a @@ -2344,71 +2494,10 @@ PMIX_EXPORT pmix_status_t PMIx_Data_copy_payload(pmix_data_buffer_t *dest, pmix_data_buffer_t *src); -/* Key-Value pair management macros */ -// TODO: add all possible types/fields here. +static inline void pmix_darray_destruct(pmix_data_array_t *m); -#define PMIX_VAL_FIELD_int(x) ((x)->data.integer) -#define PMIX_VAL_FIELD_uint32_t(x) ((x)->data.uint32) -#define PMIX_VAL_FIELD_uint16_t(x) ((x)->data.uint16) -#define PMIX_VAL_FIELD_string(x) ((x)->data.string) -#define PMIX_VAL_FIELD_float(x) ((x)->data.fval) -#define PMIX_VAL_FIELD_byte(x) ((x)->data.byte) -#define PMIX_VAL_FIELD_flag(x) ((x)->data.flag) - -#define PMIX_VAL_TYPE_int PMIX_INT -#define PMIX_VAL_TYPE_uint32_t PMIX_UINT32 -#define PMIX_VAL_TYPE_uint16_t PMIX_UINT16 -#define PMIX_VAL_TYPE_string PMIX_STRING -#define PMIX_VAL_TYPE_float PMIX_FLOAT -#define PMIX_VAL_TYPE_byte PMIX_BYTE -#define PMIX_VAL_TYPE_flag PMIX_BOOL - -#define PMIX_VAL_set_assign(_v, _field, _val ) \ - do { \ - (_v)->type = PMIX_VAL_TYPE_ ## _field; \ - PMIX_VAL_FIELD_ ## _field((_v)) = _val; \ - } while (0) - -#define PMIX_VAL_set_strdup(_v, _field, _val ) \ - do { \ - (_v)->type = PMIX_VAL_TYPE_ ## _field; \ - PMIX_VAL_FIELD_ ## _field((_v)) = strdup(_val); \ - } while (0) - -#define PMIX_VAL_SET_int PMIX_VAL_set_assign -#define PMIX_VAL_SET_uint32_t PMIX_VAL_set_assign -#define PMIX_VAL_SET_uint16_t PMIX_VAL_set_assign -#define PMIX_VAL_SET_string PMIX_VAL_set_strdup -#define PMIX_VAL_SET_float PMIX_VAL_set_assign -#define PMIX_VAL_SET_byte PMIX_VAL_set_assign -#define PMIX_VAL_SET_flag PMIX_VAL_set_assign - -#define PMIX_VAL_SET(_v, _field, _val ) \ - PMIX_VAL_SET_ ## _field(_v, _field, _val) - -#define PMIX_VAL_cmp_val(_val1, _val2) ((_val1) != (_val2)) -#define PMIX_VAL_cmp_float(_val1, _val2) (((_val1)>(_val2))?(((_val1)-(_val2))>0.000001):(((_val2)-(_val1))>0.000001)) -#define PMIX_VAL_cmp_ptr(_val1, _val2) strncmp(_val1, _val2, strlen(_val1)+1) - -#define PMIX_VAL_CMP_int PMIX_VAL_cmp_val -#define PMIX_VAL_CMP_uint32_t PMIX_VAL_cmp_val -#define PMIX_VAL_CMP_uint16_t PMIX_VAL_cmp_val -#define PMIX_VAL_CMP_float PMIX_VAL_cmp_float -#define PMIX_VAL_CMP_string PMIX_VAL_cmp_ptr -#define PMIX_VAL_CMP_byte PMIX_VAL_cmp_val -#define PMIX_VAL_CMP_flag PMIX_VAL_cmp_val - -#define PMIX_VAL_ASSIGN(_v, _field, _val) \ - PMIX_VAL_set_assign(_v, _field, _val) - -#define PMIX_VAL_CMP(_field, _val1, _val2) \ - PMIX_VAL_CMP_ ## _field(_val1, _val2) - -#define PMIX_VAL_FREE(_v) \ - PMIx_free_value_data(_v) - -static inline void pmix_value_destruct(pmix_value_t * m) { - size_t _n; +static inline void pmix_value_destruct(pmix_value_t * m) +{ if (PMIX_STRING == (m)->type) { if (NULL != (m)->data.string) { free((m)->data.string); @@ -2422,66 +2511,178 @@ static inline void pmix_value_destruct(pmix_value_t * m) { (m)->data.bo.size = 0; } } else if (PMIX_DATA_ARRAY == (m)->type) { - if (NULL != (m)->data.darray && NULL != (m)->data.darray->array) { - if (PMIX_STRING == (m)->data.darray->type) { - char **_str = (char**)(m)->data.darray->array; - for (_n=0; _n < (m)->data.darray->size; _n++) { - if (NULL != _str[_n]) { - free(_str[_n]); - } - } - } else if (PMIX_PROC_INFO == (m)->data.darray->type) { - pmix_proc_info_t *_info = - (pmix_proc_info_t*)(m)->data.darray->array; - for (_n=0; _n < (m)->data.darray->size; _n++) { - PMIX_PROC_INFO_DESTRUCT(&_info[_n]); - } - } else if (PMIX_INFO == (m)->data.darray->type) { - pmix_info_t *_info = - (pmix_info_t*)(m)->data.darray->array; - for (_n=0; _n < (m)->data.darray->size; _n++) { - pmix_value_destruct(&_info[_n].value); - } - } else if (PMIX_BYTE_OBJECT == (m)->data.darray->type) { - pmix_byte_object_t *_obj = - (pmix_byte_object_t*)(m)->data.darray->array; - for (_n=0; _n < (m)->data.darray->size; _n++) { - if (NULL != _obj[_n].bytes) { - free(_obj[_n].bytes); - } - } - } - free((m)->data.darray->array); - (m)->data.darray->array = NULL; - (m)->data.darray->size = 0; - } if (NULL != (m)->data.darray) { + pmix_darray_destruct((m)->data.darray); free((m)->data.darray); (m)->data.darray = NULL; } - /**** DEPRECATED ****/ - } else if (PMIX_INFO_ARRAY == (m)->type) { - pmix_info_t *_p = (pmix_info_t*)((m)->data.array->array); - for (_n=0; _n < (m)->data.array->size; _n++) { - if (PMIX_STRING == _p[_n].value.type) { - if (NULL != _p[_n].value.data.string) { - free(_p[_n].value.data.string); - } - } else if (PMIX_BYTE_OBJECT == _p[_n].value.type) { - if (NULL != _p[_n].value.data.bo.bytes) { - free(_p[_n].value.data.bo.bytes); - } - } else if (PMIX_PROC_INFO == _p[_n].value.type) { - PMIX_PROC_INFO_DESTRUCT(_p[_n].value.data.pinfo); - } - } - free(_p); - /********************/ } else if (PMIX_ENVAR == (m)->type) { PMIX_ENVAR_DESTRUCT(&(m)->data.envar); + } else if (PMIX_PROC == (m)->type) { + PMIX_PROC_RELEASE((m)->data.proc); } } +static inline void pmix_darray_destruct(pmix_data_array_t *m) +{ + if (NULL != m) { + if (PMIX_INFO == m->type) { + pmix_info_t *_info = (pmix_info_t*)m->array; + PMIX_INFO_FREE(_info, m->size); + } else if (PMIX_PROC == m->type) { + pmix_proc_t *_p = (pmix_proc_t*)m->array; + PMIX_PROC_FREE(_p, m->size); + } else if (PMIX_PROC_INFO == m->type) { + pmix_proc_info_t *_pi = (pmix_proc_info_t*)m->array; + PMIX_PROC_INFO_FREE(_pi, m->size); + } else if (PMIX_ENVAR == m->type) { + pmix_envar_t *_e = (pmix_envar_t*)m->array; + PMIX_ENVAR_FREE(_e, m->size); + } else if (PMIX_VALUE == m->type) { + pmix_value_t *_v = (pmix_value_t*)m->array; + PMIX_VALUE_FREE(_v, m->size); + } else if (PMIX_PDATA == m->type) { + pmix_pdata_t *_pd = (pmix_pdata_t*)m->array; + PMIX_PDATA_FREE(_pd, m->size); + } else if (PMIX_QUERY == m->type) { + pmix_query_t *_q = (pmix_query_t*)m->array; + PMIX_QUERY_FREE(_q, m->size); + } else if (PMIX_APP == m->type) { + pmix_app_t *_a = (pmix_app_t*)m->array; + PMIX_APP_FREE(_a, m->size); + } else if (PMIX_BYTE_OBJECT == m->type) { + pmix_byte_object_t *_b = (pmix_byte_object_t*)m->array; + PMIX_BYTE_OBJECT_FREE(_b, m->size); + } else if (PMIX_STRING == m->type) { + char **_s = (char**)m->array; + size_t _si; + for (_si=0; _si < m->size; _si++) { + free(_s[_si]); + } + free(m->array); + m->array = NULL; + } else { + free(m->array); + } + } +} + +#define PMIX_DATA_ARRAY_CONSTRUCT(m, n, t) \ + do { \ + (m)->type = (t); \ + (m)->size = (n); \ + if (0 < (n)) { \ + if (PMIX_INFO == (t)) { \ + PMIX_INFO_CREATE((m)->array, (n)); \ + } else if (PMIX_PROC == (t)) { \ + PMIX_PROC_CREATE((m)->array, (n)); \ + } else if (PMIX_PROC_INFO == (t)) { \ + PMIX_PROC_INFO_CREATE((m)->array, (n)); \ + } else if (PMIX_ENVAR == (t)) { \ + PMIX_ENVAR_CREATE((m)->array, (n)); \ + } else if (PMIX_VALUE == (t)) { \ + PMIX_VALUE_CREATE((m)->array, (n)); \ + } else if (PMIX_PDATA == (t)) { \ + PMIX_PDATA_CREATE((m)->array, (n)); \ + } else if (PMIX_QUERY == (t)) { \ + PMIX_QUERY_CREATE((m)->array, (n)); \ + } else if (PMIX_APP == (t)) { \ + PMIX_APP_CREATE((m)->array, (n)); \ + } else if (PMIX_BYTE_OBJECT == (t)) { \ + PMIX_BYTE_OBJECT_CREATE((m)->array, (n)); \ + } else if (PMIX_ALLOC_DIRECTIVE == (t) || \ + PMIX_PROC_STATE == (t) || \ + PMIX_PERSIST == (t) || \ + PMIX_SCOPE == (t) || \ + PMIX_DATA_RANGE == (t) || \ + PMIX_BYTE == (t) || \ + PMIX_INT8 == (t) || \ + PMIX_UINT8 == (t)) { \ + (m)->array = calloc((n), sizeof(int8_t)); \ + } else if (PMIX_STRING == (t)) { \ + (m)->array = calloc((n), sizeof(char*)); \ + } else if (PMIX_SIZE == (t)) { \ + (m)->array = calloc((n), sizeof(size_t)); \ + } else if (PMIX_PID == (t)) { \ + (m)->array = calloc((n), sizeof(pid_t)); \ + } else if (PMIX_INT == (t) || \ + PMIX_UINT == (t) || \ + PMIX_STATUS == (t)) { \ + (m)->array = calloc((n), sizeof(int)); \ + } else if (PMIX_IOF_CHANNEL == (t) || \ + PMIX_DATA_TYPE == (t) || \ + PMIX_INT16 == (t) || \ + PMIX_UINT16 == (t)) { \ + (m)->array = calloc((n), sizeof(int16_t)); \ + } else if (PMIX_PROC_RANK == (t) || \ + PMIX_INFO_DIRECTIVES == (t) || \ + PMIX_INT32 == (t) || \ + PMIX_UINT32 == (t)) { \ + (m)->array = calloc((n), sizeof(int32_t)); \ + } else if (PMIX_INT64 == (t) || \ + PMIX_UINT64 == (t)) { \ + (m)->array = calloc((n), sizeof(int64_t)); \ + } else if (PMIX_FLOAT == (t)) { \ + (m)->array = calloc((n), sizeof(float)); \ + } else if (PMIX_DOUBLE == (t)) { \ + (m)->array = calloc((n), sizeof(double)); \ + } else if (PMIX_TIMEVAL == (t)) { \ + (m)->array = calloc((n), sizeof(struct timeval)); \ + } else if (PMIX_TIME == (t)) { \ + (m)->array = calloc((n), sizeof(time_t)); \ + } \ + } else { \ + (m)->array = NULL; \ + } \ + } while(0) +#define PMIX_DATA_ARRAY_CREATE(m, n, t) \ + do { \ + (m) = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); \ + PMIX_DATA_ARRAY_CONSTRUCT((m), (n), (t)); \ + } while(0) + +#define PMIX_DATA_ARRAY_DESTRUCT(m) pmix_darray_destruct(m) + +#define PMIX_DATA_ARRAY_FREE(m) \ + do { \ + if (NULL != (m)) { \ + PMIX_DATA_ARRAY_DESTRUCT(m); \ + free((m)); \ + (m) = NULL; \ + } \ + } while(0) + + +/** + * Provide a safe version of strncpy that doesn't generate + * a ton of spurious warnings. Note that not every environment + * provides nice string functions, and we aren't concerned about + * max performance here + * + * @param dest Destination string. + * @param src Source string. + * @param len Size of the dest array - 1 + * + */ +static inline void pmix_strncpy(char *dest, const char *src, size_t len) +{ + size_t i, k; + char *new_dest = dest; + + /* use an algorithm that also protects against + * non-NULL-terminated src strings */ + for (i=0, k=0; i <= len; ++i, ++src, ++new_dest) { + ++k; + *new_dest = *src; + if ('\0' == *src) { + break; + } + } + dest[k-1] = '\0'; +} + +#include + #if defined(c_plusplus) || defined(__cplusplus) } #endif diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix_extend.h b/opal/mca/pmix/pmix3x/pmix/include/pmix_extend.h new file mode 100644 index 0000000000..f6f320abab --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix_extend.h @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Artem Y. Polyakov . + * All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer listed + * in this license in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * The copyright holders provide no reassurances that the source code + * provided does not infringe any patent, copyright, or any other + * intellectual property rights of third parties. The copyright holders + * disclaim any liability to any recipient for claims brought against + * recipient by any third party for infringement of that parties + * intellectual property rights. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $HEADER$ + */ + +#ifndef PMIx_EXTEND_H +#define PMIx_EXTEND_H + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +/* expose some functions that are resolved in the + * PMIx library, but part of a header that + * includes internal functions - we don't + * want to expose the entire header here. These + * back the associated macros included in the + * PMIx Standard + */ +void pmix_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); + +pmix_status_t pmix_value_unload(pmix_value_t *kv, void **data, size_t *sz); + +pmix_status_t pmix_value_xfer(pmix_value_t *kv, const pmix_value_t *src); + +pmix_status_t pmix_argv_append_nosize(char ***argv, const char *arg); + +pmix_status_t pmix_argv_prepend_nosize(char ***argv, const char *arg); + +pmix_status_t pmix_argv_append_unique_nosize(char ***argv, const char *arg, bool overwrite); + +void pmix_argv_free(char **argv); + +char **pmix_argv_split(const char *src_string, int delimiter); + +int pmix_argv_count(char **argv); + +char *pmix_argv_join(char **argv, int delimiter); + +char **pmix_argv_copy(char **argv); + +pmix_status_t pmix_setenv(const char *name, const char *value, + bool overwrite, char ***env); + + +/* the following are a set of legacy macros not included in the + * PMIx Standard, but used in some codes (e.g., the Slurm plugin). + * These should be considered "deprecated" and will be removed + * in the next major release of the PRI */ +#define PMIX_VAL_FIELD_int(x) ((x)->data.integer) +#define PMIX_VAL_FIELD_uint32_t(x) ((x)->data.uint32) +#define PMIX_VAL_FIELD_uint16_t(x) ((x)->data.uint16) +#define PMIX_VAL_FIELD_string(x) ((x)->data.string) +#define PMIX_VAL_FIELD_float(x) ((x)->data.fval) +#define PMIX_VAL_FIELD_byte(x) ((x)->data.byte) +#define PMIX_VAL_FIELD_flag(x) ((x)->data.flag) + +#define PMIX_VAL_TYPE_int PMIX_INT +#define PMIX_VAL_TYPE_uint32_t PMIX_UINT32 +#define PMIX_VAL_TYPE_uint16_t PMIX_UINT16 +#define PMIX_VAL_TYPE_string PMIX_STRING +#define PMIX_VAL_TYPE_float PMIX_FLOAT +#define PMIX_VAL_TYPE_byte PMIX_BYTE +#define PMIX_VAL_TYPE_flag PMIX_BOOL + +#define PMIX_VAL_set_assign(_v, _field, _val ) \ + do { \ + (_v)->type = PMIX_VAL_TYPE_ ## _field; \ + PMIX_VAL_FIELD_ ## _field((_v)) = _val; \ + } while (0) + +#define PMIX_VAL_set_strdup(_v, _field, _val ) \ + do { \ + (_v)->type = PMIX_VAL_TYPE_ ## _field; \ + PMIX_VAL_FIELD_ ## _field((_v)) = strdup(_val); \ + } while (0) + +#define PMIX_VAL_SET_int PMIX_VAL_set_assign +#define PMIX_VAL_SET_uint32_t PMIX_VAL_set_assign +#define PMIX_VAL_SET_uint16_t PMIX_VAL_set_assign +#define PMIX_VAL_SET_string PMIX_VAL_set_strdup +#define PMIX_VAL_SET_float PMIX_VAL_set_assign +#define PMIX_VAL_SET_byte PMIX_VAL_set_assign +#define PMIX_VAL_SET_flag PMIX_VAL_set_assign + +#define PMIX_VAL_SET(_v, _field, _val ) \ + PMIX_VAL_SET_ ## _field(_v, _field, _val) + +#define PMIX_VAL_cmp_val(_val1, _val2) ((_val1) != (_val2)) +#define PMIX_VAL_cmp_float(_val1, _val2) (((_val1)>(_val2))?(((_val1)-(_val2))>0.000001):(((_val2)-(_val1))>0.000001)) +#define PMIX_VAL_cmp_ptr(_val1, _val2) strncmp(_val1, _val2, strlen(_val1)+1) + +#define PMIX_VAL_CMP_int PMIX_VAL_cmp_val +#define PMIX_VAL_CMP_uint32_t PMIX_VAL_cmp_val +#define PMIX_VAL_CMP_uint16_t PMIX_VAL_cmp_val +#define PMIX_VAL_CMP_float PMIX_VAL_cmp_float +#define PMIX_VAL_CMP_string PMIX_VAL_cmp_ptr +#define PMIX_VAL_CMP_byte PMIX_VAL_cmp_val +#define PMIX_VAL_CMP_flag PMIX_VAL_cmp_val + +#define PMIX_VAL_ASSIGN(_v, _field, _val) \ + PMIX_VAL_set_assign(_v, _field, _val) + +#define PMIX_VAL_CMP(_field, _val1, _val2) \ + PMIX_VAL_CMP_ ## _field(_val1, _val2) + +#define PMIX_VAL_FREE(_v) \ + PMIx_free_value_data(_v) + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix_server.h b/opal/mca/pmix/pmix3x/pmix/include/pmix_server.h index ecc08307fe..4d3f36bbd4 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/pmix_server.h +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix_server.h @@ -596,7 +596,7 @@ PMIX_EXPORT pmix_status_t PMIx_generate_ppn(const char *input, char **ppn); * for the PMIx server library to correctly handle collectives * as a collective operation call can occur before all the * procs have been started */ -PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int nlocalprocs, +PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const pmix_nspace_t nspace, int nlocalprocs, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); @@ -605,7 +605,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int n * intended to support persistent PMIx servers by providing * an opportunity for the host RM to tell the PMIx server * library to release all memory for a completed job */ -PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[], +PMIX_EXPORT void PMIx_server_deregister_nspace(const pmix_nspace_t nspace, pmix_op_cbfunc_t cbfunc, void *cbdata); /* Register a client process with the PMIx server library. The @@ -676,7 +676,7 @@ typedef void (*pmix_setup_application_cbfunc_t)(pmix_status_t status, * operation in case network libraries need to perform some action * before responding. Any returned env will be distributed along * with the application */ -PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const char nspace[], +PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const pmix_nspace_t nspace, pmix_info_t info[], size_t ninfo, pmix_setup_application_cbfunc_t cbfunc, void *cbdata); @@ -692,7 +692,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_application(const char nspace[], * for the first local client - i.e., they will only be executed * once for a given nspace */ -PMIX_EXPORT pmix_status_t PMIx_server_setup_local_support(const char nspace[], +PMIX_EXPORT pmix_status_t PMIx_server_setup_local_support(const pmix_nspace_t nspace, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata); diff --git a/opal/mca/pmix/pmix3x/pmix/include/pmix_version.h.in b/opal/mca/pmix/pmix3x/pmix/include/pmix_version.h.in index dc8570f38f..44987a6592 100644 --- a/opal/mca/pmix/pmix3x/pmix/include/pmix_version.h.in +++ b/opal/mca/pmix/pmix3x/pmix/include/pmix_version.h.in @@ -2,7 +2,7 @@ * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2018 IBM Corporation. All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -19,5 +19,5 @@ #define PMIX_VERSION_MINOR @pmixminor@ #define PMIX_VERSION_RELEASE @pmixrelease@ -#define PMIX_NUMERIC_VERSION 0x00030001 +#define PMIX_NUMERIC_VERSION 0x00030100 #endif diff --git a/opal/mca/pmix/pmix3x/pmix/man/Makefile.am b/opal/mca/pmix/pmix3x/pmix/man/Makefile.am deleted file mode 100644 index 7c0f8bffe4..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/Makefile.am +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -if !PMIX_EMBEDDED_MODE - -man_MANS = \ - man3/pmix_init.3 \ - man3/pmix_finalize.3 \ - man3/pmix_initialized.3 \ - man3/pmix_abort.3 \ - man3/pmix_put.3 \ - man3/pmix_commit.3 \ - man7/pmix.7 \ - man7/pmix_constants.7 - -EXTRA_DIST = $(man_MANS) - -man3/pmix_init.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_init.3.md; - -man3/pmix_finalize.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_finalize.3.md; - -man3/pmix_initialized.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_initialized.3.md; - -man3/pmix_abort.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_abort.3.md; - -man3/pmix_put.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_put.3.md; - -man3/pmix_commit.3: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_commit.3.md; - -man7/pmix.7: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix.7.md; - -man7/pmix_constants.7: - $(top_srcdir)/contrib/md2nroff.pl --source=pmix_constants.7.md; - -endif # !PMIX_EMBEDDED_MODE diff --git a/opal/mca/pmix/pmix3x/pmix/man/README b/opal/mca/pmix/pmix3x/pmix/man/README deleted file mode 100644 index 73c605cb7f..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/README +++ /dev/null @@ -1,186 +0,0 @@ -This file describes how the developer side of man pages work in PMIx. - -The Definitive Source Of Truth man pages are the Markdown man pages in -this directory (i.e., the files ending in ..md. If you want to -edit man pages, you need to edit the ..md pages. Do NOT edit -the . nroff man pages directly; these files are automatically -generated -- you will lose any manual edits the next time those files -are generated. - -The Markdown web pages are rendered in two different ways: - -1. Nroff man pages. These man pages are put into the `master` branch - and later included in PMIx distribution tarballs. - -2. HTML. The http://open-mpi.github.io/pmix/ web site (which is - served by the Github web servers) automatically renders the content - of the `gh-pages` branch of the PMIx repo. - -Markdown syntax -=============== - -The definitive man pages are the Markdown man pages. To edit them, -you need to understand the syntax used in these files. - -The canonical reference for Markdown is here: - - http://daringfireball.net/projects/markdown/syntax - -Note, however, that the PMIx Markdown man pages are served via -the Github Pages web servers, which use a system called Jekyll to -render the Markdown into HTML (https://github.com/jekyll/jekyll). -As such, there are a few Jekyll annotations in the PMIx Markdown -pages (so that they can be served up properly from Github's web -servers). - -If you're familiar with Markdown, you should be ok. But there are a -small number differences and quirks with which you should be familiar: - -1. The first few lines of each file are a YAML header and include - directive for Jekyll. DO NOT REMOVE THIS HEADER (or the file will - not render to HTML properly when served up from Github's web - servers). Here's a sample YAML header from pmix.7.md: - ---- -layout: page -title: PMIx(7) -tagline: PMIx Programmer's Manual ---- -{% include JB/setup %} - - The whole block is needed, and it must be the first input in the - file. - -2. In Github-flavored Markdown, you may be used to using "fenced - blocks" for multi-line code blocks, like this: - -```c -void my_c_code(void) { - int i; - /* Hello, world */ -} -``` - - Such fenced blocks will not work in Jekyll. Instead, you must - delineate your code blocks with Jekyll delimiters: - -{% highlight c %} -void my_c_code(void) { - int i; - /* Hello, world */ -} -{% endhighlight %} - - This will result in a pretty code box in the rendered HTML output, - and it will be syntax highlighted for the C language. Leave the - "c" out of the first directive if your multi-line block is not C - code, and then it won't do C syntax highlighting. - -3. The PMIx man pages are full of 2-level lists of things. E.g., - lists of functions, and then in some of the functions, there is a - sub-list of flags that can be used with that function. - - The convention used in the PMIx man pages is to highlight a - word/phrase representing each list item. Then use a ":" to start - the next line that describes that item. For example: - -*PMIX_FLOAT* -: A single-precision floating point value (IEEE 754). - - This will make the token "PMIX_FLOAT" be highlighted in both - HTML and nroff output, and then the paragraph that comes after it - will be properly delimited and indented. - - To make a sub-list inside an item, use the same format, but prefix - the sub-list items with "-", like this: - -*scope* -: Flag that controls the visible scope of the data. - -- *PMIX_GLOBAL* -: Indicates that the data is to be visible to all applications executed - by this user. - -4. There may be a small number of places in the PMIx man pages where - there are unnumbered lists with deliberate line breaks. For - example: - -foo / bar -baz / goo -: Something really intelligent - - Note the first line is "foo / bar", and then there is - a deliberate line break, and then the second line is "baz / goo". - - To effect the deliberate line break, you have to put two blank - spaces after "bar". To show that graphically (showing "_" - for " "): - -foo / bar__ -baz / goo -: Something really intelligent - -5. The "SEE ALSO" items at the end of each man page are linked to - their corresponding man pages. Note that the links are made to - ".html" files -- *not* ".md" files. If you care, the reason is - because the Github web servers statically generate .html files from - the .md files when you git push to the gh-pages branch. Hence, the - man pages are actually served from static .html files on the Github - web servers. - - Also, since links are meaningless in nroff, they are effectively - ignored in the resulting nroff output. - -Workflow -======== - -The workflow is like this: - -1. Developer edits ..md files for new changes. - -2. In a perfect world, the developer makes perfect edits and pushes - the changes up to `master`. An automated cron job will eventually - notice the new pages, and do two things: - - 2a. Copy the modified Markdown pages to the `gh-master` branch (so - that they go live on the web site). - - 2b. Re-generate any relevant nroff man pages in `master`. - - The automated cron job actually does exist and does these things, - but it should only be relied upon once a developer is sure that - their changes to the Markdown man pages are correct. - -3. To check that the changes will render properly, developers should - do two things: - - 3a. Run "make nroff". This will convert all the Markdown man pages - into nroff man pages (in the man/ directory). Check to ensure - that your changes look appropriate in the rendered nroff - output. - - *CAUTION* The "pandoc" utility is used to generate the nroff - files from the Markdown source. Different versions of pandoc - will generate slightly different nroff output. Meaning: when - you run "make nroff", you might end up changing every nroff man - page, simply because your version of pandoc is different than - the last person who ran it. Please only check in your changes, - if possible. - - 3b. Check out the `gh-pages` branch from PMIx and copy any - modified Markdown pages into the "master/man" directory (i.e., - the directory for man pages from the master development - branch). - - Then run the "jekyll serve" command from the top-level - directory in `gh-pages`. This runs a local web server on your - computer and renders the Markdown files into HTML such that you - can point a browser to http://127.0.0.1:4000 and see the web - site. - - If you make any changes to files in the tree where "jekyll" is - running, Jekyll will notice the changes and automatically - re-generate the relevant HTML. Meaning: you can just refresh - the page from http://127.0.0.1:4000 in your browser and you'll - see your changes -- there's no need to restart Jekyll to force - it to notice new changes. diff --git a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_abort.3 b/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_abort.3 deleted file mode 100644 index ea0690ae94..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_abort.3 +++ /dev/null @@ -1,62 +0,0 @@ -.TH "pmix_abort" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Abort \- Abort the specified processes -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx\\_Abort(int\ status,\ const\ char\ msg[], -\ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ pmix\\_proc\\_t\ procs[],\ size_t\ nprocs); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]status\f[] : Status value to be returned. -A value of zero is permitted by PMIx, but may not be returned by some -resource managers. -.PP -\f[I]msg\f[] : A string message to be displayed -.PP -\f[I]procs\f[] : An array of pmix_proc_t structures defining the -processes to be aborted. -A \f[I]NULL\f[] for the proc array indicates that all processes in the -caller\[aq]s nspace are to be aborted. -A wildcard value for the rank in any structure indicates that all -processes in that nspace are to be aborted. -.PP -\f[I]nprocs\f[] : Number of pmix_proc_t structures in the \f[I]procs\f[] -array -.SH DESCRIPTION -.PP -Request that the provided array of procs be aborted, returning the -provided \f[I]status\f[] and printing the provided message. -A \f[I]NULL\f[] for the proc array indicates that all processes in the -caller\[aq]s nspace are to be aborted. -.PP -The response to this request is somewhat dependent on the specific -resource manager and its configuration (e.g., some resource managers -will not abort the application if the provided \f[I]status\f[] is zero -unless specifically configured to do so), and thus lies outside the -control of PMIx itself. -However, the client will inform the RM of the request that the -application be aborted, regardless of the value of the provided -\f[I]status\f[]. -.PP -Passing a \f[I]NULL\f[] msg parameter is allowed. -Note that race conditions caused by multiple processes calling -PMIx_Abort are left to the server implementation to resolve with regard -to which status is returned and what messages (if any) are printed. -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.SH SEE ALSO -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_commit.3 b/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_commit.3 deleted file mode 100644 index 31dd4fc2f7..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_commit.3 +++ /dev/null @@ -1,35 +0,0 @@ -.TH "pmix_commit" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Commit \- Push all previously \f[I]PMIx_Put\f[] values to the local -PMIx server. -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx_Commit(void); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]none\f[] -.SH DESCRIPTION -.PP -This is an asynchronous operation \- the library will immediately return -to the caller while the data is transmitted to the local server in the -background -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.SH SEE ALSO -.PP -\f[C]PMIx_Put\f[](3) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_finalize.3 b/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_finalize.3 deleted file mode 100644 index 6b15282945..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_finalize.3 +++ /dev/null @@ -1,45 +0,0 @@ -.TH "pmix_finalize" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Finalize \- Finalize the PMIx Client -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx\\_Finalize(const\ pmix\\_info\\_t\ info[],\ size_t\ ninfo); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]info\f[] : An optional array of pmix_info_t structures -.PP -\f[I]ninfo\f[] : Number of pmix_info_t structures in the pmix_info_t -array -.SH DESCRIPTION -.PP -Finalize the PMIx client, closing the connection with the local PMIx -server and releasing all malloc\[aq]d memory. -.PP -The info array is used to pass user requests regarding the fence -operation. -This can include: -.IP "(a)" 4 -PMIX_EMBED_BARRIER \- By default, \f[I]PMIx_Finalize\f[] does not -include an internal barrier operation. -This attribute directs \f[I]PMIx_Finalize\f[] to execute a barrier as -part of the finalize operation. -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.SH SEE ALSO -.PP -\f[C]PMIx_Init\f[](3) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_init.3 b/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_init.3 deleted file mode 100644 index b988200b33..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_init.3 +++ /dev/null @@ -1,52 +0,0 @@ -.TH "pmix_init" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Init \- Initialize the PMIx Client -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx_Init(pmix\\_proc\\_t\ *proc); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]proc\f[] : Pointer to a pmix_proc_t object in which the -client\[aq]s namespace and rank are to be returned. -.SH DESCRIPTION -.PP -Initialize the PMIx client, returning the process identifier assigned to -this client\[aq]s application in the provided pmix_proc_t struct. -Passing a value of \f[I]NULL\f[] for this parameter is allowed if the -user wishes solely to initialize the PMIx system and does not require -return of the identifier at that time. -.PP -When called, the PMIx client will check for the required connection -information of the local PMIx server and will establish the connection. -If the information is not found, or the server connection fails, then an -appropriate error constant will be returned. -.PP -If successful, the function will return PMIX_SUCCESS and will fill the -provided structure with the server\-assigned namespace and rank of the -process within the application. -In addition, all startup information provided by the resource manager -will be made available to the client process via subsequent calls to -\f[I]PMIx_Get\f[]. -.PP -Note that the PMIx client library is referenced counted, and so multiple -calls to PMIx_Init are allowed. -Thus, one way to obtain the namespace and rank of the process is to -simply call PMIx_Init with a non\-NULL parameter. -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.SH SEE ALSO -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_initialized.3 b/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_initialized.3 deleted file mode 100644 index 6e2170f35c..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_initialized.3 +++ /dev/null @@ -1,30 +0,0 @@ -.TH "pmix_initialized" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Initialized \- Check if \f[I]PMIx_Init\f[] has been called -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -int\ PMIx_Initialized(void); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]none\f[] -.SH DESCRIPTION -.PP -Check to see if the PMIx Client library has been intialized -.SH RETURN VALUE -.PP -Returns \f[I]true\f[] if the PMIx Client has been initialized, and -\f[I]false\f[] if not. -.SH ERRORS -.SH NOTES -.SH SEE ALSO -.PP -\f[C]PMIx_Init\f[](3) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_put.3 b/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_put.3 deleted file mode 100644 index e960583e40..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/man3/pmix_put.3 +++ /dev/null @@ -1,60 +0,0 @@ -.TH "pmix_put" "3" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx_Put \- Push a value into the client\[aq]s namespace -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ - -pmix\\_status\\_t\ PMIx\\_Init(pmix\\_scope\\_t\ scope,\ const\ char\ key[],\ pmix\\_value\\_t\ *val); -\f[] -.fi -.SH ARGUMENTS -.PP -\f[I]scope\f[] : Defines a scope for data "put" by PMI per the -following: -.IP "(a)" 4 -PMI_LOCAL \- the data is intended only for other application processes -on the same node. -Data marked in this way will not be included in data packages sent to -remote requestors -.IP "(b)" 4 -PMI_REMOTE \- the data is intended solely for application processes on -remote nodes. -Data marked in this way will not be shared with other processes on the -same node -.IP "(c)" 4 -PMI_GLOBAL \- the data is to be shared with all other requesting -processes, regardless of location -.PP -\f[I]key\f[] : String key identifying the information. -This can be either one of the PMIx defined attributes, or a -user\-defined value -.PP -\f[I]val\f[] : Pointer to a pmix_value_t structure containing the data -to be pushed along with the type of the provided data. -.SH DESCRIPTION -.PP -Push a value into the client\[aq]s namespace. -The client library will cache the information locally until -\f[I]PMIx_Commit\f[] is called. -The provided scope value is passed to the local PMIx server, which will -distribute the data as directed. -.SH RETURN VALUE -.PP -Returns PMIX_SUCCESS on success. -On error, a negative value corresponding to a PMIx errno is returned. -.SH ERRORS -.PP -PMIx errno values are defined in \f[C]pmix_common.h\f[]. -.SH NOTES -.PP -See \[aq]pmix_common.h\[aq] for definition of the pmix_value_t -structure. -.SH SEE ALSO -.PP -\f[C]PMIx_Constants\f[](7), \f[C]PMIx_Structures\f[](7) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix3x/pmix/man/man7/pmix.7 b/opal/mca/pmix/pmix3x/pmix/man/man7/pmix.7 deleted file mode 100644 index d2e545af94..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/man7/pmix.7 +++ /dev/null @@ -1,35 +0,0 @@ -.TH "pmix" "7" "2015\-10\-29" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -Process Management Interface \- Exascale -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ -\f[] -.fi -.SH OVERVIEW -.PP -The Process Management Interface (PMI) has been used for quite some time -as a means of exchanging wireup information needed for interprocess -communication. -Two versions (PMI\-1 and PMI\-2) have been released as part of the MPICH -effort. -While PMI\-2 demonstrates better scaling properties than its PMI\-1 -predecessor, attaining rapid launch and wireup of the roughly 1M -processes executing across 100k nodes expected for exascale operations -remains challenging. -.PP -PMI Exascale (PMIx) represents an attempt to resolve these questions by -providing an extended version of the PMI standard specifically designed -to support clusters up to and including exascale sizes. -The overall objective of the project is not to branch the existing -pseudo\-standard definitions \- in fact, PMIx fully supports both of the -existing PMI\-1 and PMI\-2 APIs \- but rather to (a) augment and extend -those APIs to eliminate some current restrictions that impact -scalability, and (b) provide a reference implementation of the -PMI\-server that demonstrates the desired level of scalability. -.SH SEE ALSO -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix3x/pmix/man/man7/pmix_constants.7 b/opal/mca/pmix/pmix3x/pmix/man/man7/pmix_constants.7 deleted file mode 100644 index bd6414cb7a..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/man/man7/pmix_constants.7 +++ /dev/null @@ -1,79 +0,0 @@ -.TH "pmix_constants" "7" "2016\-03\-01" "PMIx Programmer\[aq]s Manual" "\@VERSION\@" -.SH NAME -.PP -PMIx Constants -.SH SYNOPSIS -.IP -.nf -\f[C] -#include\ -\f[] -.fi -.SH OVERVIEW -.PP -PMIx relies on the following types of constants: -.PP -\f[I]Maximum Sizes\f[] : In order to minimize malloc performance -penalties, PMIx utilizes constant\-sized arrays wherever possible. -These constants provide the user with the maximum size of the various -array types. -.PP -\f[I]Attributes\f[] : . -.PP -\f[I]Errors\f[] : PMIx uses negative error constants, with 0 indicating -"success". -.SH MAXIMUM SIZES -.PP -The . -.PP -\f[I]PMIX_MAX_NSLEN\f[] : The maximum length of a namespace. -Note that any declaration of an array to hold a key string must include -one extra space for the terminating \f[I]NULL\f[]. -.PP -\f[I]PMIX_MAX_KEYLEN\f[] : Maximum length of the key string used in -structures such as the \f[I]pmix\f[]info_t_. -Note that any declaration of an array to hold a key string must include -one extra space for the terminating \f[I]NULL\f[]. -.SH ATTRIBUTES -.PP -Define a set of "standard" PMIx attributes that can be queried using the -PMIx_Get function. -Implementations (and users) are free to extend as desired \- thus, -functions calling PMIx_Get must be capable of handling the "not found" -condition. -Note that these are attributes of the system and the job as opposed to -values the application (or underlying programming library) might choose -to expose \- i.e., they are values provided by the resource manager as -opposed to the application. -Thus, these keys are RESERVED for use by PMIx, and users should avoid -defining any attribute starting with the keyword \f[I]PMIX\f[]. -.PP -A list of the current PMIx attributes, and the type of their associated -data value, is provided here. -.PP -\f[I]PMIX_ATTR_UNDEF (NULL)\f[] : Used to initialize an attribute field, -indicating that the attribute has not yet been assigned. -.PP -\f[I]PMIX_USERID (uint32_t)\f[] : . -.PP -\f[I]PMIX_GRPID (uint32_t)\f[] : An access domain represents a single -logical connection into a fabric. -It may map to a single physical or virtual NIC or a port. -An access domain defines the boundary across which fabric resources may -be associated. -Each access domain belongs to a single fabric domain. -.PP -\f[I]PMIX_CPUSET (char*)\f[] : . -.SH ERROR CONSTANTS -.PP -\&. -.PP -\f[I]PMIX_SUCCESS\f[] : Indicates that the operation was successful. -.PP -\f[I]PMIX_ERROR\f[] : A general error code \- an error occurred, but no -specific reason can be provided. -.SH SEE ALSO -.PP -\f[C]pmix\f[](7) -.SH AUTHORS -PMIx. diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/Makefile.include b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/Makefile.include index dbd384e3e5..20b9a3c8f8 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/Makefile.include +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/Makefile.include @@ -15,7 +15,7 @@ # reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -30,7 +30,8 @@ headers += \ atomics/sys/atomic.h \ atomics/sys/atomic_impl.h \ atomics/sys/timer.h \ - atomics/sys/cma.h + atomics/sys/cma.h \ + atomics/sys/atomic_stdc.h include atomics/sys/x86_64/Makefile.include include atomics/sys/arm/Makefile.include diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/architecture.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/architecture.h index cb386f67b3..9fa0f4eefb 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/architecture.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/architecture.h @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017 Research Organization for Information Science @@ -47,6 +47,7 @@ #define PMIX_BUILTIN_SYNC 0200 #define PMIX_BUILTIN_GCC 0202 #define PMIX_BUILTIN_NO 0203 +#define PMIX_BUILTIN_C11 0204 /* Formats */ #define PMIX_DEFAULT 1000 /* standard for given architecture */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/arm/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/arm/atomic.h index 1598547f34..43a18b2151 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/arm/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/arm/atomic.h @@ -12,9 +12,9 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -110,7 +110,7 @@ void pmix_atomic_isync(void) #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 #define PMIX_HAVE_ATOMIC_MATH_32 1 -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -138,7 +138,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -149,7 +149,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval); @@ -158,7 +158,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t #if (PMIX_ASM_SUPPORT_64BIT == 1) #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -189,7 +189,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -200,7 +200,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval); @@ -210,7 +210,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define PMIX_HAVE_ATOMIC_ADD_32 1 -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int inc) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t* v, int inc) { int32_t t, old; int tmp; @@ -231,7 +231,7 @@ static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int inc) } #define PMIX_HAVE_ATOMIC_SUB_32 1 -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int dec) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t* v, int dec) { int32_t t, old; int tmp; diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/arm64/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/arm64/atomic.h index b3df262431..1026fa5556 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/arm64/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/arm64/atomic.h @@ -12,9 +12,9 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -83,7 +83,7 @@ static inline void pmix_atomic_isync (void) * *********************************************************************/ -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -103,7 +103,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add return ret; } -static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval) +static inline int32_t pmix_atomic_swap_32(pmix_atomic_int32_t *addr, int32_t newval) { int32_t ret, tmp; @@ -122,7 +122,7 @@ static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -143,7 +143,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -165,7 +165,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t #define pmix_atomic_ll_32(addr, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + pmix_atomic_int32_t *_addr = (addr); \ int32_t _ret; \ \ __asm__ __volatile__ ("ldaxr %w0, [%1] \n" \ @@ -177,7 +177,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t #define pmix_atomic_sc_32(addr, newval, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + pmix_atomic_int32_t *_addr = (addr); \ int32_t _newval = (int32_t) newval; \ int _ret; \ \ @@ -189,7 +189,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t ret = (_ret == 0); \ } while (0) -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -210,7 +210,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add return ret; } -static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +static inline int64_t pmix_atomic_swap_64 (pmix_atomic_int64_t *addr, int64_t newval) { int64_t ret; int tmp; @@ -230,7 +230,7 @@ static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newva atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -252,7 +252,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -275,7 +275,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define pmix_atomic_ll_64(addr, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + pmix_atomic_int64_t *_addr = (addr); \ int64_t _ret; \ \ __asm__ __volatile__ ("ldaxr %0, [%1] \n" \ @@ -287,7 +287,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define pmix_atomic_sc_64(addr, newval, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + pmix_atomic_int64_t *_addr = (addr); \ int64_t _newval = (int64_t) newval; \ int _ret; \ \ @@ -300,7 +300,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t } while (0) #define PMIX_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \ - static inline type pmix_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ + static inline type pmix_atomic_fetch_ ## name ## _ ## bits (pmix_atomic_ ## type *addr, type value) \ { \ type newval, old; \ int32_t tmp; \ diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic.h index f28ee1d35a..5aca60ea36 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic.h @@ -16,7 +16,7 @@ * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -57,7 +57,13 @@ #include #include "src/atomics/sys/architecture.h" -#include "src/include/pmix_stdint.h" +#include "src/include/pmix_stdatomic.h" + +#if PMIX_ASSEMBLY_BUILTIN == PMIX_BUILTIN_C11 + +#include "atomic_stdc.h" + +#else /* !PMIX_C_HAVE__ATOMIC */ /* do some quick #define cleanup in cases where we are doing testing... */ @@ -93,7 +99,7 @@ BEGIN_C_DECLS */ struct pmix_atomic_lock_t { union { - volatile int32_t lock; /**< The lock address (an integer) */ + pmix_atomic_int32_t lock; /**< The lock address (an integer) */ volatile unsigned char sparc_lock; /**< The lock address on sparc */ char padding[sizeof(int)]; /**< Array for optional padding */ } u; @@ -148,6 +154,8 @@ enum { PMIX_ATOMIC_LOCK_LOCKED = 1 }; +#define PMIX_ATOMIC_LOCK_INIT {.u = {.lock = PMIX_ATOMIC_LOCK_UNLOCKED}} + /********************************************************************** * * Load the appropriate architecture files and set some reasonable @@ -351,19 +359,19 @@ void pmix_atomic_unlock(pmix_atomic_lock_t *lock); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, +bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, +bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, +bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #endif @@ -376,19 +384,19 @@ bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, +bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, +bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, +bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #endif @@ -400,20 +408,20 @@ bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 -static inline int32_t pmix_atomic_add_fetch_32(volatile int32_t *addr, int delta); -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int delta); -static inline int32_t pmix_atomic_and_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_or_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_sub_fetch_32(volatile int32_t *addr, int delta); -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int delta); -static inline int32_t pmix_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value); -static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_add_fetch_32(pmix_atomic_int32_t *addr, int delta); +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int delta); +static inline int32_t pmix_atomic_and_fetch_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_or_fetch_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_xor_fetch_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_sub_fetch_32(pmix_atomic_int32_t *addr, int delta); +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int delta); +static inline int32_t pmix_atomic_min_fetch_32 (pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_min_32 (pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_max_fetch_32 (pmix_atomic_int32_t *addr, int32_t value); +static inline int32_t pmix_atomic_fetch_max_32 (pmix_atomic_int32_t *addr, int32_t value); #endif /* PMIX_HAVE_ATOMIC_MATH_32 */ @@ -430,19 +438,19 @@ static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_64 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 -static inline int64_t pmix_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta); -static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta); -static inline int64_t pmix_atomic_and_fetch_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_or_fetch_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta); -static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta); -static inline int64_t pmix_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value); -static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_add_fetch_64(pmix_atomic_int64_t *addr, int64_t delta); +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta); +static inline int64_t pmix_atomic_and_fetch_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_or_fetch_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_sub_fetch_64(pmix_atomic_int64_t *addr, int64_t delta); +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta); +static inline int64_t pmix_atomic_min_fetch_64 (pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_min_64 (pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_max_fetch_64 (pmix_atomic_int64_t *addr, int64_t value); +static inline int64_t pmix_atomic_fetch_max_64 (pmix_atomic_int64_t *addr, int64_t value); #endif /* PMIX_HAVE_ATOMIC_MATH_64 */ @@ -459,7 +467,7 @@ static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t */ #if defined(DOXYGEN) || PMIX_ENABLE_DEBUG static inline size_t -pmix_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) +pmix_atomic_add_fetch_size_t(pmix_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) pmix_atomic_add_fetch_32((int32_t*) addr, delta); @@ -471,7 +479,7 @@ pmix_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) } static inline size_t -pmix_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) +pmix_atomic_fetch_add_size_t(pmix_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) pmix_atomic_fetch_add_32((int32_t*) addr, delta); @@ -483,7 +491,7 @@ pmix_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) } static inline size_t -pmix_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) +pmix_atomic_sub_fetch_size_t(pmix_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) pmix_atomic_sub_fetch_32((int32_t*) addr, delta); @@ -495,7 +503,7 @@ pmix_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) } static inline size_t -pmix_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) +pmix_atomic_fetch_sub_size_t(pmix_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) pmix_atomic_fetch_sub_32((int32_t*) addr, delta); @@ -508,15 +516,15 @@ pmix_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) #else #if SIZEOF_SIZE_T == 4 -#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_32((volatile int32_t *) addr, delta)) -#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_32((volatile int32_t *) addr, delta)) -#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_32((volatile int32_t *) addr, delta)) -#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_32((volatile int32_t *) addr, delta)) +#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_32((pmix_atomic_int32_t *) addr, delta)) +#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_32((pmix_atomic_int32_t *) addr, delta)) +#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_32((pmix_atomic_int32_t *) addr, delta)) +#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_32((pmix_atomic_int32_t *) addr, delta)) #elif SIZEOF_SIZE_T == 8 -#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_64((volatile int64_t *) addr, delta)) -#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_64((volatile int64_t *) addr, delta)) -#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_64((volatile int64_t *) addr, delta)) -#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_64((volatile int64_t *) addr, delta)) +#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_64((pmix_atomic_int64_t *) addr, delta)) +#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_64((pmix_atomic_int64_t *) addr, delta)) +#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_64((pmix_atomic_int64_t *) addr, delta)) +#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_64((pmix_atomic_int64_t *) addr, delta)) #else #error "Unknown size_t size" #endif @@ -526,20 +534,20 @@ pmix_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) /* these are always done with inline functions, so always mark as static inline */ -static inline bool pmix_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval, +static inline bool pmix_atomic_compare_exchange_strong_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool pmix_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval, +static inline bool pmix_atomic_compare_exchange_strong_acq_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool pmix_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval, +static inline bool pmix_atomic_compare_exchange_strong_rel_xx (pmix_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool pmix_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval, - void *newval); -static inline bool pmix_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval, - void *newval); -static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval, - void *newval); +static inline bool pmix_atomic_compare_exchange_strong_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); +static inline bool pmix_atomic_compare_exchange_strong_acq_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); +static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); /** * Atomic compare and set of generic type with relaxed semantics. This @@ -555,7 +563,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See pmix_atomic_compare_exchange_* for pseudo-code. */ #define pmix_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \ - pmix_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + pmix_atomic_compare_exchange_strong_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) /** @@ -572,7 +580,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See pmix_atomic_compare_exchange_acq_* for pseudo-code. */ #define pmix_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \ - pmix_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + pmix_atomic_compare_exchange_strong_acq_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) /** @@ -589,7 +597,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See pmix_atomic_compare_exchange_rel_* for pseudo-code. */ #define pmix_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \ - pmix_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + pmix_atomic_compare_exchange_strong_rel_xx( (pmix_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) @@ -597,15 +605,15 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* a #if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64) -static inline void pmix_atomic_add_xx(volatile void* addr, +static inline void pmix_atomic_add_xx(pmix_atomic_intptr_t* addr, int32_t value, size_t length); -static inline void pmix_atomic_sub_xx(volatile void* addr, +static inline void pmix_atomic_sub_xx(pmix_atomic_intptr_t* addr, int32_t value, size_t length); -static inline intptr_t pmix_atomic_add_fetch_ptr( volatile void* addr, void* delta ); -static inline intptr_t pmix_atomic_fetch_add_ptr( volatile void* addr, void* delta ); -static inline intptr_t pmix_atomic_sub_fetch_ptr( volatile void* addr, void* delta ); -static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* delta ); +static inline intptr_t pmix_atomic_add_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta ); +static inline intptr_t pmix_atomic_fetch_add_ptr( pmix_atomic_intptr_t* addr, void* delta ); +static inline intptr_t pmix_atomic_sub_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta ); +static inline intptr_t pmix_atomic_fetch_sub_ptr( pmix_atomic_intptr_t* addr, void* delta ); /** * Atomically increment the content depending on the type. This @@ -618,7 +626,7 @@ static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* del * @param delta Value to add (converted to ). */ #define pmix_atomic_add( ADDR, VALUE ) \ - pmix_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ + pmix_atomic_add_xx( (pmix_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) /** @@ -632,7 +640,7 @@ static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* del * @param delta Value to substract (converted to ). */ #define pmix_atomic_sub( ADDR, VALUE ) \ - pmix_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ + pmix_atomic_sub_xx( (pmix_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) #endif /* PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64 */ @@ -644,6 +652,8 @@ static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* del */ #include "src/atomics/sys/atomic_impl.h" +#endif /* !PMIX_C_HAVE__ATOMIC */ + END_C_DECLS #endif /* PMIX_SYS_ATOMIC_H */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic_impl.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic_impl.h index d3a3f40dd3..ee605ca8cf 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic_impl.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic_impl.h @@ -11,9 +11,9 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -41,7 +41,7 @@ #if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #if !defined(PMIX_HAVE_ATOMIC_MIN_32) -static inline int32_t pmix_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_min_32 (pmix_atomic_int32_t *addr, int32_t value) { int32_t old = *addr; do { @@ -58,7 +58,7 @@ static inline int32_t pmix_atomic_fetch_min_32 (volatile int32_t *addr, int32_t #endif /* PMIX_HAVE_ATOMIC_MIN_32 */ #if !defined(PMIX_HAVE_ATOMIC_MAX_32) -static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_max_32 (pmix_atomic_int32_t *addr, int32_t value) { int32_t old = *addr; do { @@ -74,7 +74,7 @@ static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #endif /* PMIX_HAVE_ATOMIC_MAX_32 */ #define PMIX_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \ - static inline type pmix_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ + static inline type pmix_atomic_fetch_ ## name ## _ ## bits (pmix_atomic_ ## type *addr, type value) \ { \ type oldval; \ do { \ @@ -86,7 +86,7 @@ static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #if !defined(PMIX_HAVE_ATOMIC_SWAP_32) #define PMIX_HAVE_ATOMIC_SWAP_32 1 -static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, +static inline int32_t pmix_atomic_swap_32(pmix_atomic_int32_t *addr, int32_t newval) { int32_t old = *addr; @@ -139,7 +139,7 @@ PMIX_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub) #if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #if !defined(PMIX_HAVE_ATOMIC_MIN_64) -static inline int64_t pmix_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_min_64 (pmix_atomic_int64_t *addr, int64_t value) { int64_t old = *addr; do { @@ -156,7 +156,7 @@ static inline int64_t pmix_atomic_fetch_min_64 (volatile int64_t *addr, int64_t #endif /* PMIX_HAVE_ATOMIC_MIN_64 */ #if !defined(PMIX_HAVE_ATOMIC_MAX_64) -static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_max_64 (pmix_atomic_int64_t *addr, int64_t value) { int64_t old = *addr; do { @@ -173,7 +173,7 @@ static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t #if !defined(PMIX_HAVE_ATOMIC_SWAP_64) #define PMIX_HAVE_ATOMIC_SWAP_64 1 -static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr, +static inline int64_t pmix_atomic_swap_64(pmix_atomic_int64_t *addr, int64_t newval) { int64_t old = *addr; @@ -236,15 +236,15 @@ PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub) #if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #define PMIX_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ static inline bool \ - pmix_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ + pmix_atomic_compare_exchange_strong ## semantics ## xx (pmix_atomic_intptr_t* addr, intptr_t *oldval, \ int64_t newval, const size_t length) \ { \ switch (length) { \ case 4: \ - return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ + return pmix_atomic_compare_exchange_strong_32 ((pmix_atomic_int32_t *) addr, \ (int32_t *) oldval, (int32_t) newval); \ case 8: \ - return pmix_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \ + return pmix_atomic_compare_exchange_strong_64 ((pmix_atomic_int64_t *) addr, \ (int64_t *) oldval, (int64_t) newval); \ } \ abort(); \ @@ -252,12 +252,12 @@ PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub) #elif PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #define PMIX_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ static inline bool \ - pmix_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ + pmix_atomic_compare_exchange_strong ## semantics ## xx (pmix_atomic_intptr_t* addr, intptr_t *oldval, \ int64_t newval, const size_t length) \ { \ switch (length) { \ case 4: \ - return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ + return pmix_atomic_compare_exchange_strong_32 ((pmix_atomic_int32_t *) addr, \ (int32_t *) oldval, (int32_t) newval); \ } \ abort(); \ @@ -273,16 +273,16 @@ PMIX_ATOMIC_DEFINE_CMPXCG_XX(_rel_) #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #define PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ static inline bool \ - pmix_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ + pmix_atomic_compare_exchange_strong ## semantics ## ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \ { \ - return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ + return pmix_atomic_compare_exchange_strong_32 ((pmix_atomic_int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ } #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #define PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ static inline bool \ - pmix_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ + pmix_atomic_compare_exchange_strong ## semantics ## ptr (pmix_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \ { \ - return pmix_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ + return pmix_atomic_compare_exchange_strong_64 ((pmix_atomic_int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ } #else #error "Can not define pmix_atomic_compare_exchange_strong_ptr with existing atomics" @@ -298,9 +298,9 @@ PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if (PMIX_HAVE_ATOMIC_SWAP_32 || PMIX_HAVE_ATOMIC_SWAP_64) #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SWAP_32 -#define pmix_atomic_swap_ptr(addr, value) (void *) pmix_atomic_swap_32((int32_t *) addr, (int32_t) value) +#define pmix_atomic_swap_ptr(addr, value) (intptr_t) pmix_atomic_swap_32((pmix_atomic_int32_t *) addr, (int32_t) value) #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SWAP_64 -#define pmix_atomic_swap_ptr(addr, value) (void *) pmix_atomic_swap_64((int64_t *) addr, (int64_t) value) +#define pmix_atomic_swap_ptr(addr, value) (intptr_t) pmix_atomic_swap_64((pmix_atomic_int64_t *) addr, (int64_t) value) #endif #endif /* (PMIX_HAVE_ATOMIC_SWAP_32 || PMIX_HAVE_ATOMIC_SWAP_64) */ @@ -309,15 +309,15 @@ PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_LLSC_32 -#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_32((volatile int32_t *) (addr), ret) -#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_32((volatile int32_t *) (addr), (intptr_t) (value), ret) +#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_32((pmix_atomic_int32_t *) (addr), ret) +#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_32((pmix_atomic_int32_t *) (addr), (intptr_t) (value), ret) #define PMIX_HAVE_ATOMIC_LLSC_PTR 1 #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_LLSC_64 -#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_64((volatile int64_t *) (addr), ret) -#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_64((volatile int64_t *) (addr), (intptr_t) (value), ret) +#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_64((pmix_atomic_int64_t *) (addr), ret) +#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_64((pmix_atomic_int64_t *) (addr), (intptr_t) (value), ret) #define PMIX_HAVE_ATOMIC_LLSC_PTR 1 @@ -332,18 +332,18 @@ PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64 static inline void - pmix_atomic_add_xx(volatile void* addr, int32_t value, size_t length) + pmix_atomic_add_xx(pmix_atomic_intptr_t* addr, int32_t value, size_t length) { switch( length ) { #if PMIX_HAVE_ATOMIC_ADD_32 case 4: - (void) pmix_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value ); + (void) pmix_atomic_fetch_add_32( (pmix_atomic_int32_t*)addr, (int32_t)value ); break; #endif /* PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */ #if PMIX_HAVE_ATOMIC_ADD_64 case 8: - (void) pmix_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value ); + (void) pmix_atomic_fetch_add_64( (pmix_atomic_int64_t*)addr, (int64_t)value ); break; #endif /* PMIX_HAVE_ATOMIC_ADD_64 */ default: @@ -355,18 +355,18 @@ static inline void static inline void -pmix_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) +pmix_atomic_sub_xx(pmix_atomic_intptr_t* addr, int32_t value, size_t length) { switch( length ) { #if PMIX_HAVE_ATOMIC_SUB_32 case 4: - (void) pmix_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value ); + (void) pmix_atomic_fetch_sub_32( (pmix_atomic_int32_t*)addr, (int32_t)value ); break; #endif /* PMIX_HAVE_ATOMIC_SUB_32 */ #if PMIX_HAVE_ATOMIC_SUB_64 case 8: - (void) pmix_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value ); + (void) pmix_atomic_fetch_sub_64( (pmix_atomic_int64_t*)addr, (int64_t)value ); break; #endif /* PMIX_HAVE_ATOMIC_SUB_64 */ default: @@ -377,7 +377,7 @@ pmix_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) } #define PMIX_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \ - static inline type pmix_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \ + static inline type pmix_atomic_ ## op ## _fetch_ ## suffix (pmix_atomic_ ## ptr_type *addr, type value) \ { \ return pmix_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \ } @@ -388,13 +388,13 @@ PMIX_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32) PMIX_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32) PMIX_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32) -static inline int32_t pmix_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_min_fetch_32 (pmix_atomic_int32_t *addr, int32_t value) { int32_t old = pmix_atomic_fetch_min_32 (addr, value); return old <= value ? old : value; } -static inline int32_t pmix_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_max_fetch_32 (pmix_atomic_int32_t *addr, int32_t value) { int32_t old = pmix_atomic_fetch_max_32 (addr, value); return old >= value ? old : value; @@ -407,13 +407,13 @@ PMIX_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64) PMIX_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64) PMIX_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64) -static inline int64_t pmix_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_min_fetch_64 (pmix_atomic_int64_t *addr, int64_t value) { int64_t old = pmix_atomic_fetch_min_64 (addr, value); return old <= value ? old : value; } -static inline int64_t pmix_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_max_fetch_64 (pmix_atomic_int64_t *addr, int64_t value) { int64_t old = pmix_atomic_fetch_max_64 (addr, value); return old >= value ? old : value; @@ -421,52 +421,52 @@ static inline int64_t pmix_atomic_max_fetch_64 (volatile int64_t *addr, int64_t #endif -static inline intptr_t pmix_atomic_fetch_add_ptr( volatile void* addr, +static inline intptr_t pmix_atomic_fetch_add_ptr( pmix_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_ADD_32 - return pmix_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta); + return pmix_atomic_fetch_add_32((pmix_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_ADD_64 - return pmix_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta); + return pmix_atomic_fetch_add_64((pmix_atomic_int64_t*) addr, (unsigned long) delta); #else abort (); return 0; #endif } -static inline intptr_t pmix_atomic_add_fetch_ptr( volatile void* addr, +static inline intptr_t pmix_atomic_add_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_ADD_32 - return pmix_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta); + return pmix_atomic_add_fetch_32((pmix_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_ADD_64 - return pmix_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta); + return pmix_atomic_add_fetch_64((pmix_atomic_int64_t*) addr, (unsigned long) delta); #else abort (); return 0; #endif } -static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, +static inline intptr_t pmix_atomic_fetch_sub_ptr( pmix_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SUB_32 - return pmix_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta); + return pmix_atomic_fetch_sub_32((pmix_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SUB_32 - return pmix_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta); + return pmix_atomic_fetch_sub_64((pmix_atomic_int64_t*) addr, (unsigned long) delta); #else abort(); return 0; #endif } -static inline intptr_t pmix_atomic_sub_fetch_ptr( volatile void* addr, +static inline intptr_t pmix_atomic_sub_fetch_ptr( pmix_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SUB_32 - return pmix_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta); + return pmix_atomic_sub_fetch_32((pmix_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SUB_32 - return pmix_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta); + return pmix_atomic_sub_fetch_64((pmix_atomic_int64_t*) addr, (unsigned long) delta); #else abort(); return 0; diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic_stdc.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic_stdc.h new file mode 100644 index 0000000000..7c4a608909 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/atomic_stdc.h @@ -0,0 +1,262 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* This file provides shims between the pmix atomics interface and the C11 atomics interface. It + * is intended as the first step in moving to using C11 atomics across the entire codebase. Once + * all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then + * this shim will go away and the codebase will be updated to use C11's atomic support + * directly. + * This shim contains some functions already present in atomic_impl.h because we do not include + * atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate + * definitions to be worthwhile. */ + +#if !defined(PMIX_ATOMIC_STDC_H) +#define PMIX_ATOMIC_STDC_H + +#include +#include +#include "src/include/pmix_stdint.h" + +#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1 + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 +#define PMIX_HAVE_ATOMIC_SWAP_32 1 + +#define PMIX_HAVE_ATOMIC_MATH_32 1 +#define PMIX_HAVE_ATOMIC_ADD_32 1 +#define PMIX_HAVE_ATOMIC_AND_32 1 +#define PMIX_HAVE_ATOMIC_OR_32 1 +#define PMIX_HAVE_ATOMIC_XOR_32 1 +#define PMIX_HAVE_ATOMIC_SUB_32 1 + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 +#define PMIX_HAVE_ATOMIC_SWAP_64 1 + +#define PMIX_HAVE_ATOMIC_MATH_64 1 +#define PMIX_HAVE_ATOMIC_ADD_64 1 +#define PMIX_HAVE_ATOMIC_AND_64 1 +#define PMIX_HAVE_ATOMIC_OR_64 1 +#define PMIX_HAVE_ATOMIC_XOR_64 1 +#define PMIX_HAVE_ATOMIC_SUB_64 1 + +#define PMIX_HAVE_ATOMIC_LLSC_32 0 +#define PMIX_HAVE_ATOMIC_LLSC_64 0 +#define PMIX_HAVE_ATOMIC_LLSC_PTR 0 + +#define PMIX_HAVE_ATOMIC_MIN_32 1 +#define PMIX_HAVE_ATOMIC_MAX_32 1 + +#define PMIX_HAVE_ATOMIC_MIN_64 1 +#define PMIX_HAVE_ATOMIC_MAX_64 1 + +#define PMIX_HAVE_ATOMIC_SPINLOCKS 1 + +static inline void pmix_atomic_mb (void) +{ + atomic_thread_fence (memory_order_seq_cst); +} + +static inline void pmix_atomic_wmb (void) +{ + atomic_thread_fence (memory_order_release); +} + +static inline void pmix_atomic_rmb (void) +{ + atomic_thread_fence (memory_order_acquire); +} + +#define pmix_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) + +#define pmix_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) + +#define pmix_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_acq(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed) +#define pmix_atomic_compare_exchange_strong_rel(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed) + +#define pmix_atomic_swap_32(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) +#define pmix_atomic_swap_64(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) +#define pmix_atomic_swap_ptr(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) + +#define PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \ + static inline type pmix_atomic_fetch_ ## op ##_## bits (pmix_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \ + } \ + \ + static inline type pmix_atomic_## op ## _fetch_ ## bits (pmix_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \ + } + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +) + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -) + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |) + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^) + +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &) +PMIX_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &) + +#define pmix_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed) + +static inline int32_t pmix_atomic_fetch_min_32 (pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!pmix_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int32_t pmix_atomic_fetch_max_32 (pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!pmix_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int64_t pmix_atomic_fetch_min_64 (pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!pmix_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int64_t pmix_atomic_fetch_max_64 (pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!pmix_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int32_t pmix_atomic_min_fetch_32 (pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t old = pmix_atomic_fetch_min_32 (addr, value); + return old <= value ? old : value; +} + +static inline int32_t pmix_atomic_max_fetch_32 (pmix_atomic_int32_t *addr, int32_t value) +{ + int32_t old = pmix_atomic_fetch_max_32 (addr, value); + return old >= value ? old : value; +} + +static inline int64_t pmix_atomic_min_fetch_64 (pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t old = pmix_atomic_fetch_min_64 (addr, value); + return old <= value ? old : value; +} + +static inline int64_t pmix_atomic_max_fetch_64 (pmix_atomic_int64_t *addr, int64_t value) +{ + int64_t old = pmix_atomic_fetch_max_64 (addr, value); + return old >= value ? old : value; +} + +#define PMIX_ATOMIC_LOCK_UNLOCKED false +#define PMIX_ATOMIC_LOCK_LOCKED true + +#define PMIX_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT + +typedef atomic_flag pmix_atomic_lock_t; + +/* + * Lock initialization function. It set the lock to UNLOCKED. + */ +static inline void pmix_atomic_lock_init (pmix_atomic_lock_t *lock, bool value) +{ + atomic_flag_clear (lock); +} + + +static inline int pmix_atomic_trylock (pmix_atomic_lock_t *lock) +{ + return (int) atomic_flag_test_and_set (lock); +} + + +static inline void pmix_atomic_lock(pmix_atomic_lock_t *lock) +{ + while (pmix_atomic_trylock (lock)) { + } +} + + +static inline void pmix_atomic_unlock (pmix_atomic_lock_t *lock) +{ + atomic_flag_clear (lock); +} + + +#if PMIX_HAVE_C11_CSWAP_INT128 + +/* the C11 atomic compare-exchange is lock free so use it */ +#define pmix_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#elif PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128 + +/* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */ +__pmix_attribute_always_inline__ +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr, + pmix_int128_t *oldval, pmix_int128_t newval) +{ + pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); + bool ret = prev == *oldval; + *oldval = prev; + return ret; +} + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#else + +#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0 + +#endif + +#endif /* !defined(PMIX_ATOMIC_STDC_H) */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/cma.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/cma.h index 3eacce2335..967d13c63f 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/cma.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/cma.h @@ -4,7 +4,7 @@ * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ */ @@ -85,13 +85,13 @@ #elif PMIX_ASSEMBLY_ARCH == PMIX_S390 -#define __NR_process_vm_readv 340 -#define __NR_process_vm_writev 341 +#define __NR_process_vm_readv 340 +#define __NR_process_vm_writev 341 #elif PMIX_ASSEMBLY_ARCH == PMIX_S390X -#define __NR_process_vm_readv 340 -#define __NR_process_vm_writev 341 +#define __NR_process_vm_readv 340 +#define __NR_process_vm_writev 341 #else #error "Unsupported architecture for process_vm_readv and process_vm_writev syscalls" diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/gcc_builtin/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/gcc_builtin/atomic.h index b1f1a1c832..e07c4d8819 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/gcc_builtin/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/gcc_builtin/atomic.h @@ -11,13 +11,13 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. * Copyright (c) 2018 Triad National Security, LLC. All rights * reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -86,103 +86,103 @@ static inline void pmix_atomic_wmb(void) /* * Suppress numerous (spurious ?) warnings from Oracle Studio compilers * see https://community.oracle.com/thread/3968347 - */ + */ #if defined(__SUNPRO_C) || defined(__SUNPRO_CC) #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L) #endif -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_swap_32 (volatile int32_t *addr, int32_t newval) +static inline int32_t pmix_atomic_swap_32 (pmix_atomic_int32_t *addr, int32_t newval) { int32_t oldval; __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); return oldval; } -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int32_t delta) { return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int32_t delta) { return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); } -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +static inline int64_t pmix_atomic_swap_64 (pmix_atomic_int64_t *addr, int64_t newval) { int64_t oldval; __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); return oldval; } -static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta) { return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta) { return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } @@ -191,7 +191,7 @@ static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 -static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, @@ -204,7 +204,7 @@ static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128 /* __atomic version is not lock-free so use legacy __sync version */ -static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_pmix_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) { pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/ia32/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/ia32/atomic.h index 4e8a6d8145..c50c162a6e 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/ia32/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/ia32/atomic.h @@ -13,9 +13,9 @@ * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -85,7 +85,7 @@ static inline void pmix_atomic_isync(void) *********************************************************************/ #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { unsigned char ret; __asm__ __volatile__ ( @@ -107,15 +107,15 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add #define PMIX_HAVE_ATOMIC_SWAP_32 1 -static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr, - int32_t newval) +static inline int32_t pmix_atomic_swap_32( pmix_atomic_int32_t *addr, + int32_t newval) { int32_t oldval; __asm__ __volatile__("xchg %1, %0" : - "=r" (oldval), "=m" (*addr) : - "0" (newval), "m" (*addr) : - "memory"); + "=r" (oldval), "=m" (*addr) : + "0" (newval), "m" (*addr) : + "memory"); return oldval; } @@ -131,7 +131,7 @@ static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -151,7 +151,7 @@ static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int i) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t* v, int i) { int ret = -i; __asm__ __volatile__( diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/powerpc/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/powerpc/atomic.h index e5ce266308..cfb46eb519 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/powerpc/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/powerpc/atomic.h @@ -13,7 +13,7 @@ * Copyright (c) 2010-2017 IBM Corporation. All rights reserved. * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -145,7 +145,7 @@ void pmix_atomic_isync(void) #define PMIX_ASM_VALUE64(x) x #endif -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev; bool ret; @@ -171,7 +171,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add * load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */ #define pmix_atomic_ll_32(addr, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + pmix_atomic_int32_t *_addr = (addr); \ int32_t _ret; \ __asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \ : "=&r" (_ret) \ @@ -182,7 +182,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add #define pmix_atomic_sc_32(addr, value, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + pmix_atomic_int32_t *_addr = (addr); \ int32_t _ret, _foo, _newval = (int32_t) value; \ \ __asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \ @@ -201,7 +201,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -212,13 +212,13 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval); } -static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval) +static inline int32_t pmix_atomic_swap_32(pmix_atomic_int32_t *addr, int32_t newval) { int32_t ret; @@ -240,7 +240,7 @@ static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval #if PMIX_GCC_INLINE_ASSEMBLY #define PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \ -static inline int64_t pmix_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \ +static inline int64_t pmix_atomic_fetch_ ## type ## _64(pmix_atomic_int64_t* v, int64_t val) \ { \ int64_t t, old; \ \ @@ -262,7 +262,7 @@ PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or) PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor) PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf) -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; bool ret; @@ -285,7 +285,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #define pmix_atomic_ll_64(addr, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + pmix_atomic_int64_t *_addr = (addr); \ int64_t _ret; \ __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \ : "=&r" (_ret) \ @@ -296,7 +296,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #define pmix_atomic_sc_64(addr, value, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + pmix_atomic_int64_t *_addr = (addr); \ int64_t _foo, _newval = (int64_t) value; \ int32_t _ret; \ \ @@ -311,7 +311,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add ret = _ret; \ } while (0) -static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr, int64_t newval) +static inline int64_t pmix_atomic_swap_64(pmix_atomic_int64_t *addr, int64_t newval) { int64_t ret; @@ -336,7 +336,7 @@ static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr, int64_t newval #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int ret; @@ -383,7 +383,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -394,7 +394,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval); @@ -402,7 +402,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \ -static inline int32_t pmix_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \ +static inline int32_t pmix_atomic_fetch_ ## type ## _32(pmix_atomic_int32_t* v, int val) \ { \ int32_t t, old; \ \ diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sparcv9/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sparcv9/atomic.h index 45afeea54d..e74a152090 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sparcv9/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sparcv9/atomic.h @@ -13,9 +13,9 @@ * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -32,7 +32,7 @@ #define ASI_P "0x80" -#define MEMBAR(type) __asm__ __volatile__ ("membar " type : : : "memory") +#define MEPMIXMBAR(type) __asm__ __volatile__ ("membar " type : : : "memory") /********************************************************************** @@ -56,19 +56,19 @@ static inline void pmix_atomic_mb(void) { - MEMBAR("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad"); + MEPMIXMBAR("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad"); } static inline void pmix_atomic_rmb(void) { - MEMBAR("#LoadLoad"); + MEPMIXMBAR("#LoadLoad"); } static inline void pmix_atomic_wmb(void) { - MEMBAR("#StoreStore"); + MEPMIXMBAR("#StoreStore"); } static inline void pmix_atomic_isync(void) @@ -86,7 +86,7 @@ static inline void pmix_atomic_isync(void) *********************************************************************/ #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -108,7 +108,7 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add } -static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -119,7 +119,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval); @@ -128,7 +128,7 @@ static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t #if PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64 -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -150,7 +150,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #else /* PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64 */ -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -180,7 +180,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #endif /* PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64 */ -static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_acq_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -191,7 +191,7 @@ static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_rel_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { pmix_atomic_wmb(); return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval); diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sync_builtin/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sync_builtin/atomic.h index 2c91411e19..240d297f7a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sync_builtin/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/sync_builtin/atomic.h @@ -11,11 +11,11 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -58,7 +58,7 @@ static inline void pmix_atomic_wmb(void) #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); bool ret = prev == *oldval; @@ -72,31 +72,31 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add #define PMIX_HAVE_ATOMIC_MATH_32 1 #define PMIX_HAVE_ATOMIC_ADD_32 1 -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t *addr, int32_t delta) { return __sync_fetch_and_add(addr, delta); } #define PMIX_HAVE_ATOMIC_AND_32 1 -static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_and_32(pmix_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_and(addr, value); } #define PMIX_HAVE_ATOMIC_OR_32 1 -static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_or_32(pmix_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_or(addr, value); } #define PMIX_HAVE_ATOMIC_XOR_32 1 -static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +static inline int32_t pmix_atomic_fetch_xor_32(pmix_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_xor(addr, value); } #define PMIX_HAVE_ATOMIC_SUB_32 1 -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t *addr, int32_t delta) { return __sync_fetch_and_sub(addr, delta); } @@ -105,7 +105,7 @@ static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int32_t d #define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); bool ret = prev == *oldval; @@ -118,31 +118,31 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #define PMIX_HAVE_ATOMIC_MATH_64 1 #define PMIX_HAVE_ATOMIC_ADD_64 1 -static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t *addr, int64_t delta) { return __sync_fetch_and_add(addr, delta); } #define PMIX_HAVE_ATOMIC_AND_64 1 -static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_and_64(pmix_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_and(addr, value); } #define PMIX_HAVE_ATOMIC_OR_64 1 -static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_or_64(pmix_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_or(addr, value); } #define PMIX_HAVE_ATOMIC_XOR_64 1 -static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +static inline int64_t pmix_atomic_fetch_xor_64(pmix_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_xor(addr, value); } #define PMIX_HAVE_ATOMIC_SUB_64 1 -static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t *addr, int64_t delta) { return __sync_fetch_and_sub(addr, delta); } @@ -150,7 +150,7 @@ static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d #endif #if PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128 -static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) { pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/timer.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/timer.h index b2a562e421..e3fdc1326e 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/timer.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/timer.h @@ -13,7 +13,7 @@ * Copyright (c) 2016 Broadcom Limited. All rights reserved. * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/x86_64/atomic.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/x86_64/atomic.h index f2144eb046..005d2d66c2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/x86_64/atomic.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/x86_64/atomic.h @@ -11,11 +11,11 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. - * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -83,7 +83,7 @@ static inline void pmix_atomic_isync(void) *********************************************************************/ #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool pmix_atomic_compare_exchange_strong_32 (pmix_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { unsigned char ret; __asm__ __volatile__ ( @@ -103,13 +103,13 @@ static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *add #if PMIX_GCC_INLINE_ASSEMBLY -static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool pmix_atomic_compare_exchange_strong_64 (pmix_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { unsigned char ret; __asm__ __volatile__ ( SMPLOCK "cmpxchgq %3,%2 \n\t" "sete %0 \n\t" - : "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr)) + : "=qm" (ret), "+a" (*oldval), "+m" (*((pmix_atomic_long_t *)addr)) : "q"(newval) : "memory", "cc" ); @@ -124,7 +124,7 @@ static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *add #if PMIX_GCC_INLINE_ASSEMBLY && PMIX_HAVE_CMPXCHG16B && HAVE_PMIX_INT128_T -static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) +static inline bool pmix_atomic_compare_exchange_strong_128 (pmix_atomic_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval) { unsigned char ret; @@ -151,15 +151,15 @@ static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128 #define PMIX_HAVE_ATOMIC_SWAP_64 1 -static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr, - int32_t newval) +static inline int32_t pmix_atomic_swap_32( pmix_atomic_int32_t *addr, + int32_t newval) { int32_t oldval; __asm__ __volatile__("xchg %1, %0" : - "=r" (oldval), "+m" (*addr) : - "0" (newval) : - "memory"); + "=r" (oldval), "+m" (*addr) : + "0" (newval) : + "memory"); return oldval; } @@ -167,15 +167,15 @@ static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr, #if PMIX_GCC_INLINE_ASSEMBLY -static inline int64_t pmix_atomic_swap_64( volatile int64_t *addr, +static inline int64_t pmix_atomic_swap_64( pmix_atomic_int64_t *addr, int64_t newval) { int64_t oldval; __asm__ __volatile__("xchgq %1, %0" : - "=r" (oldval), "+m" (*addr) : - "0" (newval) : - "memory"); + "=r" (oldval), "+m" (*addr) : + "0" (newval) : + "memory"); return oldval; } @@ -197,7 +197,7 @@ static inline int64_t pmix_atomic_swap_64( volatile int64_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i) +static inline int32_t pmix_atomic_fetch_add_32(pmix_atomic_int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -218,7 +218,7 @@ static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i) * * Atomically adds @i to @v. */ -static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t* v, int64_t i) +static inline int64_t pmix_atomic_fetch_add_64(pmix_atomic_int64_t* v, int64_t i) { int64_t ret = i; __asm__ __volatile__( @@ -239,7 +239,7 @@ static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t* v, int64_t i) * * Atomically subtracts @i from @v. */ -static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int i) +static inline int32_t pmix_atomic_fetch_sub_32(pmix_atomic_int32_t* v, int i) { int ret = -i; __asm__ __volatile__( @@ -260,7 +260,7 @@ static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t* v, int64_t i) +static inline int64_t pmix_atomic_fetch_sub_64(pmix_atomic_int64_t* v, int64_t i) { int64_t ret = -i; __asm__ __volatile__( diff --git a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/x86_64/timer.h b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/x86_64/timer.h index 6b25f9aafe..31054235a5 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/x86_64/timer.h +++ b/opal/mca/pmix/pmix3x/pmix/src/atomics/sys/x86_64/timer.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2016 Los Alamos National Security, LLC. ALl rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow diff --git a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h index 02a246bfdd..883a2c5c6c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h +++ b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_hotel.h @@ -2,7 +2,7 @@ /* * Copyright (c) 2012-2016 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -190,6 +190,7 @@ static inline pmix_status_t pmix_hotel_checkin(pmix_hotel_t *hotel, /* Do we have any rooms available? */ if (PMIX_UNLIKELY(hotel->last_unoccupied_room < 0)) { + *room_num = -1; return PMIX_ERR_OUT_OF_RESOURCE; } @@ -247,6 +248,10 @@ static inline void pmix_hotel_checkout(pmix_hotel_t *hotel, int room_num) /* Bozo check */ assert(room_num < hotel->num_rooms); + if (0 > room_num) { + /* occupant wasn't checked in */ + return; + } /* If there's an occupant in the room, check them out */ room = &(hotel->rooms[room_num]); @@ -285,6 +290,11 @@ static inline void pmix_hotel_checkout_and_return_occupant(pmix_hotel_t *hotel, /* Bozo check */ assert(room_num < hotel->num_rooms); + if (0 > room_num) { + /* occupant wasn't checked in */ + *occupant = NULL; + return; + } /* If there's an occupant in the room, check them out */ room = &(hotel->rooms[room_num]); @@ -339,6 +349,10 @@ static inline void pmix_hotel_knock(pmix_hotel_t *hotel, int room_num, void **oc assert(room_num < hotel->num_rooms); *occupant = NULL; + if (0 > room_num) { + /* occupant wasn't checked in */ + return; + } /* If there's an occupant in the room, have them come to the door */ room = &(hotel->rooms[room_num]); diff --git a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_list.h b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_list.h index df3f6a2280..f29bea22a9 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_list.h +++ b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_list.h @@ -13,7 +13,7 @@ * Copyright (c) 2007 Voltaire All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -109,7 +109,7 @@ struct pmix_list_item_t #if PMIX_ENABLE_DEBUG /** Atomic reference count for debugging */ - volatile int32_t pmix_list_item_refcount; + pmix_atomic_int32_t pmix_list_item_refcount; /** The list this item belong to */ volatile struct pmix_list_t* pmix_list_item_belong_to; #endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_object.h b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_object.h index 8ad7f90f10..7b7ed41a72 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/class/pmix_object.h +++ b/opal/mca/pmix/pmix3x/pmix/src/class/pmix_object.h @@ -192,7 +192,7 @@ struct pmix_object_t { uint64_t obj_magic_id; #endif pmix_class_t *obj_class; /**< class descriptor */ - volatile int32_t obj_reference_count; /**< reference count */ + pmix_atomic_int32_t obj_reference_count; /**< reference count */ #if PMIX_ENABLE_DEBUG const char* cls_init_file_name; /**< In debug mode store the file where the object get contructed */ int cls_init_lineno; /**< In debug mode store the line number where the object get contructed */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmi1.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmi1.c index 1c8517ea0e..dd5cb66e7d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmi1.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmi1.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -75,7 +75,7 @@ PMIX_EXPORT int PMI_Init(int *spawned) *spawned = 0; } pmi_singleton = true; - (void)strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN); + pmix_strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN); myproc.rank = 0; pmi_init = 1; return PMI_SUCCESS; @@ -242,7 +242,7 @@ PMIX_EXPORT int PMI_KVS_Get( const char kvsname[], const char key[], char value[ proc.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_ANL_MAP, NULL, 0, &val) && (NULL != val) && (PMIX_STRING == val->type)) { - strncpy(value, val->data.string, length); + pmix_strncpy(value, val->data.string, length-1); PMIX_VALUE_FREE(val, 1); return PMI_SUCCESS; } else { @@ -259,7 +259,7 @@ PMIX_EXPORT int PMI_KVS_Get( const char kvsname[], const char key[], char value[ /* retrieve the data from PMIx - since we don't have a rank, * we indicate that by passing the UNDEF value */ - (void)strncpy(proc.nspace, kvsname, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, kvsname, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_UNDEF; rc = PMIx_Get(&proc, key, NULL, 0, &val); @@ -267,7 +267,7 @@ PMIX_EXPORT int PMI_KVS_Get( const char kvsname[], const char key[], char value[ if (PMIX_STRING != val->type) { rc = PMIX_ERROR; } else if (NULL != val->data.string) { - (void)strncpy(value, val->data.string, length); + pmix_strncpy(value, val->data.string, length-1); } PMIX_VALUE_RELEASE(val); } @@ -445,7 +445,7 @@ PMIX_EXPORT int PMI_Publish_name(const char service_name[], const char port[]) } /* pass the service/port */ - (void) strncpy(info.key, service_name, PMIX_MAX_KEYLEN); + pmix_strncpy(info.key, service_name, PMIX_MAX_KEYLEN); info.value.type = PMIX_STRING; info.value.data.string = (char*) port; @@ -497,7 +497,7 @@ PMIX_EXPORT int PMI_Lookup_name(const char service_name[], char port[]) PMIX_PDATA_CONSTRUCT(&pdata); /* pass the service */ - (void) strncpy(pdata.key, service_name, PMIX_MAX_KEYLEN); + pmix_strncpy(pdata.key, service_name, PMIX_MAX_KEYLEN); /* PMI-1 doesn't want the nspace back */ if (PMIX_SUCCESS != (rc = PMIx_Lookup(&pdata, 1, NULL, 0))) { @@ -514,7 +514,7 @@ PMIX_EXPORT int PMI_Lookup_name(const char service_name[], char port[]) * potential we could overrun it. As this feature * isn't widely supported in PMI-1, try being * conservative */ - (void) strncpy(port, pdata.value.data.string, PMIX_MAX_KEYLEN); + pmix_strncpy(port, pdata.value.data.string, PMIX_MAX_KEYLEN); PMIX_PDATA_DESTRUCT(&pdata); return PMIX_SUCCESS; @@ -535,7 +535,7 @@ PMIX_EXPORT int PMI_Get_id(char id_str[], int length) return PMI_ERR_INVALID_LENGTH; } - (void) strncpy(id_str, myproc.nspace, length); + pmix_strncpy(id_str, myproc.nspace, length-1); return PMI_SUCCESS; } @@ -742,7 +742,7 @@ PMIX_EXPORT int PMI_Spawn_multiple(int count, apps[i].info = (pmix_info_t*)malloc(apps[i].ninfo * sizeof(pmix_info_t)); /* copy the info objects */ for (j = 0; j < apps[i].ninfo; j++) { - (void)strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN); + pmix_strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN); apps[i].info[j].value.type = PMIX_STRING; apps[i].info[j].value.data.string = strdup(info_keyval_vectors[i][j].val); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c index 2ad443c738..4adf6ba37d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmi2.c @@ -79,7 +79,7 @@ PMIX_EXPORT int PMI2_Init(int *spawned, int *size, int *rank, int *appnum) *appnum = 0; } pmi2_singleton = true; - (void)strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN); + pmix_strncpy(myproc.nspace, "1234", PMIX_MAX_NSLEN); myproc.rank = 0; pmi2_init = 1; return PMI2_SUCCESS; @@ -227,7 +227,7 @@ PMIX_EXPORT int PMI2_Job_Spawn(int count, const char * cmds[], apps[i].info = (pmix_info_t*)malloc(apps[i].ninfo * sizeof(pmix_info_t)); /* copy the info objects */ for (j=0; j < apps[i].ninfo; j++) { - (void)strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN); + pmix_strncpy(apps[i].info[j].key, info_keyval_vectors[i][j].key, PMIX_MAX_KEYLEN); apps[i].info[j].value.type = PMIX_STRING; apps[i].info[j].value.data.string = strdup(info_keyval_vectors[i][j].val); } @@ -271,7 +271,7 @@ PMIX_EXPORT int PMI2_Job_GetId(char jobid[], int jobid_size) if (NULL == jobid) { return PMI2_ERR_INVALID_ARGS; } - (void)strncpy(jobid, myproc.nspace, jobid_size); + pmix_strncpy(jobid, myproc.nspace, jobid_size-1); return PMI2_SUCCESS; } @@ -339,7 +339,7 @@ PMIX_EXPORT int PMI2_Job_Connect(const char jobid[], PMI2_Connect_comm_t *conn) } memset(proc.nspace, 0, sizeof(proc.nspace)); - (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1); + pmix_strncpy(proc.nspace, (jobid ? jobid : proc.nspace), PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; rc = PMIx_Connect(&proc, 1, NULL, 0); return convert_err(rc); @@ -357,7 +357,7 @@ PMIX_EXPORT int PMI2_Job_Disconnect(const char jobid[]) } memset(proc.nspace, 0, sizeof(proc.nspace)); - (void)strncpy(proc.nspace, (jobid ? jobid : proc.nspace), sizeof(proc.nspace)-1); + pmix_strncpy(proc.nspace, (jobid ? jobid : proc.nspace), PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; rc = PMIx_Disconnect(&proc, 1, NULL, 0); return convert_err(rc); @@ -455,7 +455,7 @@ PMIX_EXPORT int PMI2_KVS_Get(const char *jobid, int src_pmi_id, pmix_output_verbose(3, pmix_globals.debug_output, "PMI2_KVS_Get: key=%s jobid=%s src_pmi_id=%d", key, (jobid ? jobid : "null"), src_pmi_id); - (void)strncpy(proc.nspace, (jobid ? jobid : myproc.nspace), PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, (jobid ? jobid : myproc.nspace), PMIX_MAX_NSLEN); if (src_pmi_id == PMI2_ID_NULL) { /* the rank is UNDEF */ proc.rank = PMIX_RANK_UNDEF; @@ -468,7 +468,7 @@ PMIX_EXPORT int PMI2_KVS_Get(const char *jobid, int src_pmi_id, if (PMIX_STRING != val->type) { rc = PMIX_ERROR; } else if (NULL != val->data.string) { - (void)strncpy(value, val->data.string, maxvalue); + pmix_strncpy(value, val->data.string, maxvalue-1); *vallen = strlen(val->data.string); } PMIX_VALUE_RELEASE(val); @@ -511,7 +511,7 @@ PMIX_EXPORT int PMI2_Info_GetNodeAttr(const char name[], if (PMIX_STRING != val->type) { rc = PMIX_ERROR; } else if (NULL != val->data.string) { - (void)strncpy(value, val->data.string, valuelen); + pmix_strncpy(value, val->data.string, valuelen-1); *found = 1; } PMIX_VALUE_RELEASE(val); @@ -586,7 +586,7 @@ PMIX_EXPORT int PMI2_Info_GetJobAttr(const char name[], char value[], int valuel proc.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS == PMIx_Get(&proc, PMIX_ANL_MAP, NULL, 0, &val) && (NULL != val) && (PMIX_STRING == val->type)) { - strncpy(value, val->data.string, valuelen); + pmix_strncpy(value, val->data.string, valuelen); PMIX_VALUE_FREE(val, 1); *found = 1; return PMI2_SUCCESS; @@ -610,7 +610,7 @@ PMIX_EXPORT int PMI2_Info_GetJobAttr(const char name[], char value[], int valuel if (PMIX_STRING != val->type) { rc = PMIX_ERROR; } else if (NULL != val->data.string) { - (void)strncpy(value, val->data.string, valuelen); + pmix_strncpy(value, val->data.string, valuelen-1); *found = 1; } PMIX_VALUE_RELEASE(val); @@ -648,14 +648,14 @@ PMIX_EXPORT int PMI2_Nameserv_publish(const char service_name[], } /* pass the service/port */ - (void)strncpy(info[0].key, service_name, PMIX_MAX_KEYLEN); + pmix_strncpy(info[0].key, service_name, PMIX_MAX_KEYLEN); info[0].value.type = PMIX_STRING; info[0].value.data.string = (char*)port; nvals = 1; /* if provided, add any other value */ if (NULL != info_ptr) { - (void)strncpy(info[1].key, info_ptr->key, PMIX_MAX_KEYLEN); + pmix_strncpy(info[1].key, info_ptr->key, PMIX_MAX_KEYLEN); info[1].value.type = PMIX_STRING; info[1].value.data.string = (char*)info_ptr->val; nvals = 2; @@ -689,12 +689,12 @@ PMIX_EXPORT int PMI2_Nameserv_lookup(const char service_name[], PMIX_PDATA_CONSTRUCT(&pdata[1]); /* pass the service */ - (void)strncpy(pdata[0].key, service_name, PMIX_MAX_KEYLEN); + pmix_strncpy(pdata[0].key, service_name, PMIX_MAX_KEYLEN); nvals = 1; /* if provided, add any other value */ if (NULL != info_ptr) { - (void)strncpy(pdata[1].key, info_ptr->key, PMIX_MAX_KEYLEN); + pmix_strncpy(pdata[1].key, info_ptr->key, PMIX_MAX_KEYLEN); pdata[1].value.type = PMIX_STRING; pdata[1].value.data.string = info_ptr->val; nvals = 2; @@ -716,7 +716,7 @@ PMIX_EXPORT int PMI2_Nameserv_lookup(const char service_name[], } /* return the port */ - (void)strncpy(port, pdata[0].value.data.string, portLen); + pmix_strncpy(port, pdata[0].value.data.string, portLen-1); PMIX_PDATA_DESTRUCT(&pdata[0]); if (NULL != info_ptr) { diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c index 3e4c9433fb..a99f714192 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client.c @@ -53,7 +53,7 @@ #include PMIX_EVENT2_THREAD_HEADER static const char pmix_version_string[] = PMIX_VERSION; - +static pmix_status_t pmix_init_result = PMIX_ERR_INIT; #include "src/class/pmix_list.h" #include "src/event/pmix_event.h" @@ -268,6 +268,9 @@ static void notification_fn(size_t evhdlr_registration_id, char *name = NULL; size_t n; + pmix_output_verbose(2, pmix_client_globals.base_output, + "[%s:%d] DEBUGGER RELEASE RECVD", + pmix_globals.myid.nspace, pmix_globals.myid.rank); if (NULL != info) { lock = NULL; for (n=0; n < ninfo; n++) { @@ -439,7 +442,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, * rank should be known. So return them here if * requested */ if (NULL != proc) { - (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); proc->rank = pmix_globals.myid.rank; } ++pmix_globals.init_cntr; @@ -450,10 +453,13 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, if (NULL != info) { _check_for_notify(info, ninfo); } - return PMIX_SUCCESS; + return pmix_init_result; } + ++pmix_globals.init_cntr; + /* if we don't see the required info, then we cannot init */ if (NULL == (evar = getenv("PMIX_NAMESPACE"))) { + pmix_init_result = PMIX_ERR_INVALID_NAMESPACE; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INVALID_NAMESPACE; } @@ -463,6 +469,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_CLIENT, info, ninfo, pmix_client_notify_recv))) { PMIX_ERROR_LOG(rc); + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -480,24 +487,25 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_pointer_array_init(&pmix_client_globals.peers, 1, INT_MAX, 1); pmix_client_globals.myserver = PMIX_NEW(pmix_peer_t); if (NULL == pmix_client_globals.myserver) { + pmix_init_result = PMIX_ERR_NOMEM; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == pmix_client_globals.myserver->nptr) { PMIX_RELEASE(pmix_client_globals.myserver); + pmix_init_result = PMIX_ERR_NOMEM; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); if (NULL == pmix_client_globals.myserver->info) { PMIX_RELEASE(pmix_client_globals.myserver); + pmix_init_result = PMIX_ERR_NOMEM; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } - pmix_output_verbose(2, pmix_client_globals.base_output, - "pmix: init called"); /* setup the base verbosity */ if (0 < pmix_client_globals.base_verbose) { /* set default output */ @@ -506,17 +514,21 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_client_globals.base_verbose); } + pmix_output_verbose(2, pmix_client_globals.base_output, + "pmix: init called"); + /* we require our nspace */ if (NULL != proc) { - (void)strncpy(proc->nspace, evar, PMIX_MAX_NSLEN); + pmix_strncpy(proc->nspace, evar, PMIX_MAX_NSLEN); } - (void)strncpy(pmix_globals.myid.nspace, evar, PMIX_MAX_NSLEN); - /* set the global pmix_nspace_t object for our peer */ + PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, evar); + /* set the global pmix_namespace_t object for our peer */ pmix_globals.mypeer->nptr->nspace = strdup(evar); /* we also require our rank */ if (NULL == (evar = getenv("PMIX_RANK"))) { /* let the caller know that the server isn't available yet */ + pmix_init_result = PMIX_ERR_DATA_VALUE_NOT_FOUND; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_DATA_VALUE_NOT_FOUND; } @@ -528,6 +540,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, /* setup a rank_info object for us */ pmix_globals.mypeer->info = PMIX_NEW(pmix_rank_info_t); if (NULL == pmix_globals.mypeer->info) { + pmix_init_result = PMIX_ERR_NOMEM; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } @@ -540,6 +553,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, evar = getenv("PMIX_SECURITY_MODE"); pmix_globals.mypeer->nptr->compat.psec = pmix_psec_base_assign_module(evar); if (NULL == pmix_globals.mypeer->nptr->compat.psec) { + pmix_init_result = PMIX_ERR_INIT; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } @@ -574,6 +588,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_client_globals.myserver->nptr->compat.gds = pmix_gds_base_assign_module(NULL, 0); } if (NULL == pmix_client_globals.myserver->nptr->compat.gds) { + pmix_init_result = PMIX_ERR_INIT; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } @@ -596,6 +611,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_globals.mypeer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1); if (NULL == pmix_globals.mypeer->nptr->compat.gds) { PMIX_INFO_DESTRUCT(&ginfo); + pmix_init_result = PMIX_ERR_INIT; PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_INIT; } @@ -604,6 +620,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, /* connect to the server */ rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo); if (PMIX_SUCCESS != rc) { + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -619,6 +636,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(req); + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -627,6 +645,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, req, job_data, (void*)&cb); if (PMIX_SUCCESS != rc) { + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } @@ -636,15 +655,16 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, PMIX_DESTRUCT(&cb); if (PMIX_SUCCESS == rc) { - pmix_globals.init_cntr++; + pmix_init_result = PMIX_SUCCESS; } else { + pmix_init_result = rc; PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } PMIX_RELEASE_THREAD(&pmix_global_lock); /* look for a debugger attach key */ - (void)strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); wildcard.rank = PMIX_RANK_WILDCARD; PMIX_INFO_LOAD(&ginfo, PMIX_OPTIONAL, NULL, PMIX_BOOL); if (PMIX_SUCCESS == PMIx_Get(&wildcard, PMIX_DEBUG_STOP_IN_INIT, &ginfo, 1, &val)) { @@ -655,7 +675,9 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, PMIX_CONSTRUCT_LOCK(&releaselock); PMIX_INFO_LOAD(&evinfo[0], PMIX_EVENT_RETURN_OBJECT, &releaselock, PMIX_POINTER); PMIX_INFO_LOAD(&evinfo[1], PMIX_EVENT_HDLR_NAME, "WAIT-FOR-DEBUGGER", PMIX_STRING); - + pmix_output_verbose(2, pmix_client_globals.base_output, + "[%s:%d] WAITING IN INIT FOR DEBUGGER", + pmix_globals.myid.nspace, pmix_globals.myid.rank); PMIx_Register_event_handler(&code, 1, evinfo, 2, notification_fn, evhandler_reg_callbk, (void*)®lock); /* wait for registration to complete */ @@ -1003,7 +1025,9 @@ static void _putfn(int sd, short args, void *cbdata) PMIX_WAKEUP_THREAD(&cb->lock); } -PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, const char key[], pmix_value_t *val) +PMIX_EXPORT pmix_status_t PMIx_Put(pmix_scope_t scope, + const pmix_key_t key, + pmix_value_t *val) { pmix_cb_t *cb; pmix_status_t rc; @@ -1212,7 +1236,7 @@ static void _resolve_peers(int sd, short args, void *cbdata) /* need to thread-shift this request */ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, - const char *nspace, + const pmix_nspace_t nspace, pmix_proc_t **procs, size_t *nprocs) { pmix_cb_t *cb; @@ -1239,7 +1263,7 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_peers(const char *nodename, /* if the nspace wasn't found, then we need to * ask the server for that info */ if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; /* any key will suffice as it will bring down * the entire data blob */ @@ -1285,7 +1309,7 @@ static void _resolve_nodes(int fd, short args, void *cbdata) } /* need to thread-shift this request */ -PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist) +PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const pmix_nspace_t nspace, char **nodelist) { pmix_cb_t *cb; pmix_status_t rc; @@ -1309,7 +1333,7 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist /* if the nspace wasn't found, then we need to * ask the server for that info */ if (PMIX_ERR_INVALID_NAMESPACE == cb->status) { - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; /* any key will suffice as it will bring down * the entire data blob */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_fence.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_fence.c index d3e182d584..7a587dcad4 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_fence.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_fence.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2014-2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -147,7 +147,7 @@ PMIX_EXPORT pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs /* if we are given a NULL proc, then the caller is referencing * all procs within our own nspace */ if (NULL == procs) { - (void)strncpy(rg.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(rg.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); rg.rank = PMIX_RANK_WILDCARD; rgs = &rg; nrg = 1; diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c index 04a2e09184..46d6c62db8 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_get.c @@ -5,7 +5,7 @@ * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . * All rights reserved. - * Copyright (c) 2016 Mellanox Technologies, Inc. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -75,8 +75,15 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, static void _value_cbfunc(pmix_status_t status, pmix_value_t *kv, void *cbdata); +static pmix_status_t _getfn_fastpath(const pmix_proc_t *proc, const pmix_key_t key, + const pmix_info_t info[], size_t ninfo, + pmix_value_t **val); -PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], +static pmix_status_t process_values(pmix_value_t **v, pmix_cb_t *cb); + + +PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, + const pmix_key_t key, const pmix_info_t info[], size_t ninfo, pmix_value_t **val) { @@ -91,12 +98,17 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], } PMIX_RELEASE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_client_globals.get_verbose, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix:client get for %s:%d key %s", (NULL == proc) ? "NULL" : proc->nspace, (NULL == proc) ? PMIX_RANK_UNDEF : proc->rank, (NULL == key) ? "NULL" : key); + /* try to get data directly, without threadshift */ + if (PMIX_SUCCESS == (rc = _getfn_fastpath(proc, key, info, ninfo, val))) { + goto done; + } + /* create a callback object as we need to pass it to the * recv routine so we know which callback to use when * the return message is recvd */ @@ -115,13 +127,14 @@ PMIX_EXPORT pmix_status_t PMIx_Get(const pmix_proc_t *proc, const char key[], } PMIX_RELEASE(cb); - pmix_output_verbose(2, pmix_client_globals.get_verbose, + done: + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix:client get completed"); return rc; } -PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key, +PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const pmix_key_t key, const pmix_info_t info[], size_t ninfo, pmix_value_cbfunc_t cbfunc, void *cbdata) { @@ -149,7 +162,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key, * Either case is supported. However, we don't currently * support the case where -both- values are NULL */ if (NULL == proc && NULL == key) { - pmix_output_verbose(2, pmix_client_globals.get_verbose, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix: get_nb value error - both proc and key are NULL"); return PMIX_ERR_BAD_PARAM; } @@ -157,7 +170,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key, /* if the key is NULL, the rank cannot be WILDCARD as * we cannot return all info from every rank */ if (NULL != proc && PMIX_RANK_WILDCARD == proc->rank && NULL == key) { - pmix_output_verbose(2, pmix_client_globals.get_verbose, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix: get_nb value error - WILDCARD rank and key is NULL"); return PMIX_ERR_BAD_PARAM; } @@ -179,7 +192,7 @@ PMIX_EXPORT pmix_status_t PMIx_Get_nb(const pmix_proc_t *proc, const char *key, rank = proc->rank; } - pmix_output_verbose(2, pmix_client_globals.get_verbose, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix: get_nb value for proc %s:%u key %s", nm, rank, (NULL == key) ? "NULL" : key); @@ -284,7 +297,7 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_proc_t proc; pmix_kval_t *kv; - pmix_output_verbose(2, pmix_client_globals.get_verbose, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix: get_nb callback recvd"); if (NULL == cb) { @@ -294,7 +307,7 @@ static void _getnb_cbfunc(struct pmix_peer_t *pr, } /* cache the proc id */ - (void)strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); proc.rank = cb->pname.rank; /* a zero-byte buffer indicates that this recv is being @@ -407,7 +420,7 @@ static pmix_status_t process_values(pmix_value_t **v, pmix_cb_t *cb) /* copy the list elements */ n=0; PMIX_LIST_FOREACH(kv, kvs, pmix_kval_t) { - (void)strncpy(info[n].key, kv->key, PMIX_MAX_KEYLEN); + pmix_strncpy(info[n].key, kv->key, PMIX_MAX_KEYLEN); pmix_value_xfer(&info[n].value, kv->value); ++n; } @@ -472,6 +485,55 @@ static void infocb(pmix_status_t status, } } +static pmix_status_t _getfn_fastpath(const pmix_proc_t *proc, const pmix_key_t key, + const pmix_info_t info[], size_t ninfo, + pmix_value_t **val) +{ + pmix_cb_t *cb = PMIX_NEW(pmix_cb_t); + pmix_status_t rc = PMIX_SUCCESS; + size_t n; + + /* scan the incoming directives */ + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_DATA_SCOPE, PMIX_MAX_KEYLEN)) { + cb->scope = info[n].value.data.scope; + break; + } + } + } + cb->proc = (pmix_proc_t*)proc; + cb->copy = true; + cb->key = (char*)key; + cb->info = (pmix_info_t*)info; + cb->ninfo = ninfo; + + PMIX_GDS_FETCH_IS_TSAFE(rc, pmix_globals.mypeer); + if (PMIX_SUCCESS == rc) { + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, cb); + if (PMIX_SUCCESS == rc) { + goto done; + } + } + PMIX_GDS_FETCH_IS_TSAFE(rc, pmix_client_globals.myserver); + if (PMIX_SUCCESS == rc) { + PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb); + if (PMIX_SUCCESS == rc) { + goto done; + } + } + PMIX_RELEASE(cb); + return rc; + + done: + rc = process_values(val, cb); + if (NULL != *val) { + PMIX_VALUE_COMPRESSED_STRING_UNPACK(*val); + } + PMIX_RELEASE(cb); + return rc; +} + static void _getnbfn(int fd, short flags, void *cbdata) { pmix_cb_t *cb = (pmix_cb_t*)cbdata; @@ -480,7 +542,6 @@ static void _getnbfn(int fd, short flags, void *cbdata) pmix_value_t *val = NULL; pmix_status_t rc; size_t n; - char *tmp; pmix_proc_t proc; bool optional = false; bool immediate = false; @@ -490,13 +551,13 @@ static void _getnbfn(int fd, short flags, void *cbdata) /* cb was passed to us from another thread - acquire it */ PMIX_ACQUIRE_OBJECT(cb); - pmix_output_verbose(2, pmix_client_globals.get_verbose, + pmix_output_verbose(2, pmix_client_globals.get_output, "pmix: getnbfn value for proc %s:%u key %s", cb->pname.nspace, cb->pname.rank, (NULL == cb->key) ? "NULL" : cb->key); /* set the proc object identifier */ - (void)strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, cb->pname.nspace, PMIX_MAX_NSLEN); proc.rank = cb->pname.rank; /* scan the incoming directives */ @@ -528,12 +589,12 @@ static void _getnbfn(int fd, short flags, void *cbdata) cb->copy = true; PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, cb); if (PMIX_SUCCESS == rc) { - pmix_output_verbose(5, pmix_client_globals.get_verbose, + pmix_output_verbose(5, pmix_client_globals.get_output, "pmix:client data found in internal storage"); rc = process_values(&val, cb); goto respond; } - pmix_output_verbose(5, pmix_client_globals.get_verbose, + pmix_output_verbose(5, pmix_client_globals.get_output, "pmix:client data NOT found in internal storage"); /* if the key is NULL or starts with "pmix", then they are looking @@ -545,7 +606,7 @@ static void _getnbfn(int fd, short flags, void *cbdata) cb->copy = true; PMIX_GDS_FETCH_KV(rc, pmix_client_globals.myserver, cb); if (PMIX_SUCCESS != rc) { - pmix_output_verbose(5, pmix_client_globals.get_verbose, + pmix_output_verbose(5, pmix_client_globals.get_output, "pmix:client job-level data NOT found"); if (0 != strncmp(cb->pname.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) { /* we are asking about the job-level info from another @@ -559,7 +620,7 @@ static void _getnbfn(int fd, short flags, void *cbdata) * job-level info. In some cases, a server may elect not * to provide info at init to save memory */ if (immediate) { - pmix_output_verbose(5, pmix_client_globals.get_verbose, + pmix_output_verbose(5, pmix_client_globals.get_output, "pmix:client IMMEDIATE given - querying data"); /* the direct modex request doesn't pass a key as it * was intended to support non-job-level information. @@ -579,16 +640,16 @@ static void _getnbfn(int fd, short flags, void *cbdata) return; } /* we should have had this info, so respond with the error */ - pmix_output_verbose(5, pmix_client_globals.get_verbose, + pmix_output_verbose(5, pmix_client_globals.get_output, "pmix:client returning NOT FOUND error"); goto respond; } else { - pmix_output_verbose(5, pmix_client_globals.get_verbose, + pmix_output_verbose(5, pmix_client_globals.get_output, "pmix:client NULL KEY - returning error"); goto respond; } } - pmix_output_verbose(5, pmix_client_globals.get_verbose, + pmix_output_verbose(5, pmix_client_globals.get_output, "pmix:client job-level data NOT found"); rc = process_values(&val, cb); goto respond; @@ -608,19 +669,7 @@ static void _getnbfn(int fd, short flags, void *cbdata) /* if a callback was provided, execute it */ if (NULL != cb->cbfunc.valuefn) { if (NULL != val) { - /* if this is a compressed string, then uncompress it */ - if (PMIX_COMPRESSED_STRING == val->type) { - pmix_util_uncompress_string(&tmp, (uint8_t*)val->data.bo.bytes, val->data.bo.size); - if (NULL == tmp) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - rc = PMIX_ERR_NOMEM; - PMIX_VALUE_RELEASE(val); - val = NULL; - } else { - PMIX_VALUE_DESTRUCT(val); - PMIX_VAL_ASSIGN(val, string, tmp); - } - } + PMIX_VALUE_COMPRESSED_STRING_UNPACK(val); } cb->cbfunc.valuefn(rc, val, cb->cbdata); } @@ -644,7 +693,7 @@ static void _getnbfn(int fd, short flags, void *cbdata) * us to attempt to retrieve it from the server */ if (optional) { /* they don't want us to try and retrieve it */ - pmix_output_verbose(2, pmix_client_globals.get_verbose, + pmix_output_verbose(2, pmix_client_globals.get_output, "PMIx_Get key=%s for rank = %u, namespace = %s was not found - request was optional", cb->key, cb->pname.rank, cb->pname.nspace); rc = PMIX_ERR_NOT_FOUND; @@ -672,7 +721,7 @@ static void _getnbfn(int fd, short flags, void *cbdata) goto respond; } - pmix_output_verbose(2, pmix_client_globals.get_verbose, + pmix_output_verbose(2, pmix_client_globals.get_output, "%s:%d REQUESTING DATA FROM SERVER FOR %s:%d KEY %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, cb->pname.nspace, cb->pname.rank, cb->key); diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_pub.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_pub.c index cee3dcaf94..bd6795eac5 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_pub.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_pub.c @@ -660,7 +660,7 @@ static void lookup_cbfunc(pmix_status_t status, pmix_pdata_t pdata[], size_t nda for (j=0; j < cb->nvals; j++) { if (0 == strcmp(pdata[i].key, tgt[j].key)) { /* transfer the publishing proc id */ - (void)strncpy(tgt[j].proc.nspace, pdata[i].proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(tgt[j].proc.nspace, pdata[i].proc.nspace, PMIX_MAX_NSLEN); tgt[j].proc.rank = pdata[i].proc.rank; /* transfer the value to the pmix_info_t */ PMIX_BFROPS_VALUE_XFER(cb->status, pmix_client_globals.myserver, &tgt[j].value, &pdata[i].value); diff --git a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_spawn.c b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_spawn.c index f04dcec34d..b7aefc4316 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_spawn.c +++ b/opal/mca/pmix/pmix3x/pmix/src/client/pmix_client_spawn.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -49,6 +49,7 @@ #include "src/class/pmix_list.h" #include "src/threads/threads.h" #include "src/mca/bfrops/bfrops.h" +#include "src/mca/pnet/base/base.h" #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/output.h" @@ -64,7 +65,7 @@ static void spawn_cbfunc(pmix_status_t status, char nspace[], void *cbdata); PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, const pmix_app_t apps[], size_t napps, - char nspace[]) + pmix_nspace_t nspace) { pmix_status_t rc; pmix_cb_t *cb; @@ -104,7 +105,7 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, PMIX_WAIT_THREAD(&cb->lock); rc = cb->status; if (NULL != nspace) { - (void)strncpy(nspace, cb->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(nspace, cb->pname.nspace, PMIX_MAX_NSLEN); } PMIX_RELEASE(cb); @@ -119,6 +120,12 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin pmix_cmd_t cmd = PMIX_SPAWNNB_CMD; pmix_status_t rc; pmix_cb_t *cb; + size_t n, m; + pmix_app_t *aptr; + bool jobenvars = false; + char *harvest[2] = {"PMIX_MCA_", NULL}; + pmix_kval_t *kv; + pmix_list_t ilist; PMIX_ACQUIRE_THREAD(&pmix_global_lock); @@ -137,6 +144,70 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin } PMIX_RELEASE_THREAD(&pmix_global_lock); + /* check job info for directives */ + if (NULL != job_info) { + for (n=0; n < ninfo; n++) { + if (PMIX_CHECK_KEY(&job_info[n], PMIX_SETUP_APP_ENVARS)) { + PMIX_CONSTRUCT(&ilist, pmix_list_t); + rc = pmix_pnet_base_harvest_envars(harvest, NULL, &ilist); + if (PMIX_SUCCESS != rc) { + PMIX_LIST_DESTRUCT(&ilist); + return rc; + } + PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) { + /* cycle across all the apps and set this envar */ + for (m=0; m < napps; m++) { + aptr = (pmix_app_t*)&apps[m]; + pmix_setenv(kv->value->data.envar.envar, + kv->value->data.envar.value, + true, &aptr->env); + } + } + jobenvars = true; + PMIX_LIST_DESTRUCT(&ilist); + break; + } + } + } + + for (n=0; n < napps; n++) { + /* do a quick check of the apps directive array to ensure + * the ninfo field has been set */ + aptr = (pmix_app_t*)&apps[n]; + if (NULL != aptr->info && 0 == aptr->ninfo) { + /* look for the info marked as "end" */ + m = 0; + while (!(PMIX_INFO_IS_END(&aptr->info[m])) && m < SIZE_MAX) { + ++m; + } + if (SIZE_MAX == m) { + /* nothing we can do */ + return PMIX_ERR_BAD_PARAM; + } + aptr->ninfo = m; + } + if (!jobenvars) { + for (m=0; m < aptr->ninfo; m++) { + if (PMIX_CHECK_KEY(&aptr->info[m], PMIX_SETUP_APP_ENVARS)) { + PMIX_CONSTRUCT(&ilist, pmix_list_t); + rc = pmix_pnet_base_harvest_envars(harvest, NULL, &ilist); + if (PMIX_SUCCESS != rc) { + PMIX_LIST_DESTRUCT(&ilist); + return rc; + } + PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) { + pmix_setenv(kv->value->data.envar.envar, + kv->value->data.envar.value, + true, &aptr->env); + } + jobenvars = true; + PMIX_LIST_DESTRUCT(&ilist); + break; + } + } + } + } + msg = PMIX_NEW(pmix_buffer_t); /* pack the cmd */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, @@ -240,27 +311,25 @@ static void wait_cbfunc(struct pmix_peer_t *pr, PMIX_ERROR_LOG(rc); ret = rc; } - if (PMIX_SUCCESS == ret) { - /* unpack the namespace */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, - buf, &n2, &cnt, PMIX_STRING); + /* unpack the namespace */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_client_globals.myserver, + buf, &n2, &cnt, PMIX_STRING); + if (PMIX_SUCCESS != rc && PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { + PMIX_ERROR_LOG(rc); + ret = rc; + } + pmix_output_verbose(1, pmix_globals.debug_output, + "pmix:client recv '%s'", n2); + + if (NULL != n2) { + /* protect length */ + pmix_strncpy(nspace, n2, PMIX_MAX_NSLEN); + free(n2); + PMIX_GDS_STORE_JOB_INFO(rc, pmix_globals.mypeer, nspace, buf); + /* extract and process any job-related info for this nspace */ if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - ret = rc; - } - pmix_output_verbose(1, pmix_globals.debug_output, - "pmix:client recv '%s'", n2); - - if (NULL != n2) { - /* protect length */ - (void)strncpy(nspace, n2, PMIX_MAX_NSLEN); - free(n2); - PMIX_GDS_STORE_JOB_INFO(rc, pmix_globals.mypeer, nspace, buf); - /* extract and process any job-related info for this nspace */ - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - } } } diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c index 615db82630..1c2f74308a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_control.c @@ -4,6 +4,8 @@ * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -105,6 +107,8 @@ static void query_cbfunc(struct pmix_peer_t *peer, /* release the caller */ if (NULL != cd->cbfunc) { cd->cbfunc(results->status, results->info, results->ninfo, cd->cbdata, relcbfunc, results); + } else { + PMIX_RELEASE(results); } PMIX_RELEASE(cd); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c index 39ca2bcf29..a9e32c661f 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_data.c @@ -98,7 +98,7 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) /* didn't find it, so try to get the library version of the target * from the host - the result will be cached, so we will only have * to retrieve it once */ - (void)strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); wildcard.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS != (rc = PMIx_Get(&wildcard, PMIX_BFROPS_MODULE, NULL, 0, &value))) { /* couldn't get it - nothing we can do */ @@ -110,7 +110,7 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) PMIX_RELEASE(value); return NULL; } - peer->nptr = PMIX_NEW(pmix_nspace_t); + peer->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == peer->nptr) { PMIX_RELEASE(peer); PMIX_RELEASE(value); @@ -137,15 +137,9 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) return pmix_client_globals.myserver; } - /* if the target is another member of my nspace, then - * they must be using the same version */ - if (0 == strncmp(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN)) { - return pmix_globals.mypeer; - } - /* try to get the library version of this peer - the result will be * cached, so we will only have to retrieve it once */ - (void)strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(wildcard.nspace, proc->nspace, PMIX_MAX_NSLEN); wildcard.rank = PMIX_RANK_WILDCARD; if (PMIX_SUCCESS != (rc = PMIx_Get(&wildcard, PMIX_BFROPS_MODULE, NULL, 0, &value))) { /* couldn't get it - nothing we can do */ @@ -157,7 +151,7 @@ static pmix_peer_t* find_peer(const pmix_proc_t *proc) PMIX_RELEASE(value); return NULL; } - peer->nptr = PMIX_NEW(pmix_nspace_t); + peer->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == peer->nptr) { PMIX_RELEASE(peer); PMIX_RELEASE(value); diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h index fcc5f7b3da..3525c5fb47 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_iof.h @@ -148,7 +148,7 @@ pmix_iof_fd_always_ready(int fd) "defining endpt: file %s line %d fd %d", \ __FILE__, __LINE__, (fid))); \ PMIX_CONSTRUCT((snk), pmix_iof_sink_t); \ - (void)strncpy((snk)->name.nspace, (nm)->nspace, PMIX_MAX_NSLEN); \ + pmix_strncpy((snk)->name.nspace, (nm)->nspace, PMIX_MAX_NSLEN); \ (snk)->name.rank = (nm)->rank; \ (snk)->tag = (tg); \ if (0 <= (fid)) { \ diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c index 33bc025daf..c40f08b6a0 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_query.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -53,6 +53,8 @@ static void query_cbfunc(struct pmix_peer_t *peer, pmix_status_t rc; pmix_shift_caddy_t *results; int cnt; + size_t n; + pmix_kval_t *kv; pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query cback from server"); @@ -88,6 +90,19 @@ static void query_cbfunc(struct pmix_peer_t *peer, results->status = rc; goto complete; } + /* locally cache the results */ + for (n=0; n < results->ninfo; n++) { + kv = PMIX_NEW(pmix_kval_t); + kv->key = strdup(results->info[n].key); + PMIX_VALUE_CREATE(kv->value, 1); + PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, + kv->value, &results->info[n].value); + + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, PMIX_INTERNAL, + kv); + PMIX_RELEASE(kv); // maintain accounting + } } complete: @@ -100,6 +115,22 @@ static void query_cbfunc(struct pmix_peer_t *peer, PMIX_RELEASE(cd); } +static void _local_relcb(void *cbdata) +{ + pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata; + PMIX_RELEASE(cd); +} + +static void _local_cbfunc(int sd, short args, void *cbdata) +{ + pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata; + if (NULL != cd->cbfunc) { + cd->cbfunc(cd->status, cd->info, cd->ninfo, cd->cbdata, _local_relcb, cd); + return; + } + PMIX_RELEASE(cd); +} + PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nqueries, pmix_info_cbfunc_t cbfunc, void *cbdata) @@ -108,6 +139,11 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque pmix_cmd_t cmd = PMIX_QUERY_CMD; pmix_buffer_t *msg; pmix_status_t rc; + pmix_cb_t cb; + size_t n, p; + pmix_list_t results; + pmix_kval_t *kv, *kvnxt; + pmix_proc_t proc; PMIX_ACQUIRE_THREAD(&pmix_global_lock); @@ -124,6 +160,128 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque return PMIX_ERR_BAD_PARAM; } + /* do a quick check of the qualifiers array to ensure + * the nqual field has been set */ + for (n=0; n < nqueries; n++) { + if (NULL != queries[n].qualifiers && 0 == queries[n].nqual) { + /* look for the info marked as "end" */ + p = 0; + while (!(PMIX_INFO_IS_END(&queries[n].qualifiers[p])) && p < SIZE_MAX) { + ++p; + } + if (SIZE_MAX == p) { + /* nothing we can do */ + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_BAD_PARAM; + } + queries[n].nqual = p; + } + } + + /* setup the list of local results */ + PMIX_CONSTRUCT(&results, pmix_list_t); + + /* check the directives to see if they want us to refresh + * the local cached results - if we wanted to optimize this + * more, we would check each query and allow those that don't + * want to be refreshed to be executed locally, and those that + * did would be sent to the host. However, for now we simply + * */ + memset(proc.nspace, 0, PMIX_MAX_NSLEN+1); + proc.rank = PMIX_RANK_INVALID; + for (n=0; n < nqueries; n++) { + for (p=0; p < queries[n].nqual; p++) { + if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_QUERY_REFRESH_CACHE)) { + if (PMIX_INFO_TRUE(&queries[n].qualifiers[p])) { + PMIX_LIST_DESTRUCT(&results); + goto query; + } + } else if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_PROCID)) { + PMIX_LOAD_NSPACE(proc.nspace, queries[n].qualifiers[p].value.data.proc->nspace); + proc.rank = queries[n].qualifiers[p].value.data.proc->rank; + } else if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_NSPACE)) { + PMIX_LOAD_NSPACE(proc.nspace, queries[n].qualifiers[p].value.data.string); + } else if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_RANK)) { + proc.rank = queries[n].qualifiers[p].value.data.rank; + } else if (PMIX_CHECK_KEY(&queries[n].qualifiers[p], PMIX_HOSTNAME)) { + if (0 != strcmp(queries[n].qualifiers[p].value.data.string, pmix_globals.hostname)) { + /* asking about a different host, so ask for the info */ + PMIX_LIST_DESTRUCT(&results); + goto query; + } + } + } + /* we get here if a refresh isn't required - first try a local + * "get" on the data to see if we already have it */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + cb.copy = false; + /* set the proc */ + if (PMIX_RANK_INVALID == proc.rank && + 0 == strlen(proc.nspace)) { + /* use our id */ + cb.proc = &pmix_globals.myid; + } else { + if (0 == strlen(proc.nspace)) { + /* use our nspace */ + PMIX_LOAD_NSPACE(cb.proc->nspace, pmix_globals.myid.nspace); + } + if (PMIX_RANK_INVALID == proc.rank) { + /* user the wildcard rank */ + proc.rank = PMIX_RANK_WILDCARD; + } + cb.proc = &proc; + } + for (p=0; NULL != queries[n].keys[p]; p++) { + cb.key = queries[n].keys[p]; + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); + if (PMIX_SUCCESS != rc) { + /* needs to be passed to the host */ + PMIX_LIST_DESTRUCT(&results); + PMIX_DESTRUCT(&cb); + goto query; + } + /* need to retain this result */ + PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &cb.kvs, pmix_kval_t) { + pmix_list_remove_item(&cb.kvs, &kv->super); + pmix_list_append(&results, &kv->super); + } + PMIX_DESTRUCT(&cb); + } + } + + /* if we get here, then all queries were completely locally + * resolved, so construct the results for return */ + cd = PMIX_NEW(pmix_query_caddy_t); + cd->cbfunc = cbfunc; + cd->cbdata = cbdata; + cd->status = PMIX_SUCCESS; + cd->ninfo = pmix_list_get_size(&results); + PMIX_INFO_CREATE(cd->info, cd->ninfo); + n = 0; + PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &results, pmix_kval_t) { + PMIX_LOAD_KEY(cd->info[n].key, kv->key); + rc = pmix_value_xfer(&cd->info[n].value, kv->value); + if (PMIX_SUCCESS != rc) { + cd->status = rc; + PMIX_INFO_FREE(cd->info, cd->ninfo); + break; + } + ++n; + } + /* done with the list of results */ + PMIX_LIST_DESTRUCT(&results); + /* we need to thread-shift as we are not allowed to + * execute the callback function prior to returning + * from the API */ + PMIX_THREADSHIFT(cd, _local_cbfunc); + /* regardless of the result of the query, we return + * PMIX_SUCCESS here to indicate that the operation + * was accepted for processing */ + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_SUCCESS; + + + query: /* if we are the server, then we just issue the query and * return the response */ if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && @@ -135,10 +293,10 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque } pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query handed to RM"); - pmix_host_server.query(&pmix_globals.myid, - queries, nqueries, - cbfunc, cbdata); - return PMIX_SUCCESS; + rc = pmix_host_server.query(&pmix_globals.myid, + queries, nqueries, + cbfunc, cbdata); + return rc; } /* if we aren't connected, don't attempt to send */ @@ -177,6 +335,7 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque PMIX_RELEASE(cd); return rc; } + pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query sending to server"); PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, diff --git a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c index 7ee50b50fa..48a2d5c7a1 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c +++ b/opal/mca/pmix/pmix3x/pmix/src/common/pmix_strings.c @@ -12,6 +12,8 @@ * Copyright (c) 2007-2012 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -241,25 +243,24 @@ PMIX_EXPORT const char* PMIx_IOF_channel_string(pmix_iof_channel_t channel) { size_t cnt=0; - memset(answer, 0, sizeof(answer)); if (PMIX_FWD_STDIN_CHANNEL & channel) { - strncpy(&answer[cnt], "STDIN ", strlen("STDIN ")); + strcpy(&answer[cnt], "STDIN "); cnt += strlen("STDIN "); } if (PMIX_FWD_STDOUT_CHANNEL & channel) { - strncpy(&answer[cnt], "STDOUT ", strlen("STDOUT ")); + strcpy(&answer[cnt], "STDOUT "); cnt += strlen("STDOUT "); } if (PMIX_FWD_STDERR_CHANNEL & channel) { - strncpy(&answer[cnt], "STDERR ", strlen("STDERR ")); + strcpy(&answer[cnt], "STDERR "); cnt += strlen("STDERR "); } if (PMIX_FWD_STDDIAG_CHANNEL & channel) { - strncpy(&answer[cnt], "STDDIAG ", strlen("STDDIAG ")); + strcpy(&answer[cnt], "STDDIAG "); cnt += strlen("STDDIAG "); } if (0 == cnt) { - strncpy(&answer[cnt], "NONE", strlen("NONE")); + strcpy(&answer[cnt], "NONE"); } return answer; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h index 1cd7d3fe71..6ba6b77493 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h +++ b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event.h @@ -172,63 +172,84 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status, void pmix_event_timeout_cb(int fd, short flags, void *arg); -#define PMIX_REPORT_EVENT(e, p, r, f) \ - do { \ - pmix_event_chain_t *ch, *cp; \ - size_t n, ninfo; \ - pmix_info_t *info; \ - pmix_proc_t proc; \ - \ - ch = NULL; \ - /* see if we already have this event cached */ \ - PMIX_LIST_FOREACH(cp, &pmix_globals.cached_events, pmix_event_chain_t) { \ - if (cp->status == (e)) { \ - ch = cp; \ - break; \ - } \ - } \ - if (NULL == ch) { \ - /* nope - need to add it */ \ - ch = PMIX_NEW(pmix_event_chain_t); \ - ch->status = (e); \ - ch->range = (r); \ - (void)strncpy(ch->source.nspace, \ - (p)->nptr->nspace, \ - PMIX_MAX_NSLEN); \ - ch->source.rank = (p)->info->pname.rank; \ - ch->ninfo = 0; \ - ch->nallocated = 2; \ - ch->final_cbfunc = (f); \ - ch->final_cbdata = ch; \ - PMIX_INFO_CREATE(ch->info, ch->nallocated); \ - /* cache it */ \ - pmix_list_append(&pmix_globals.cached_events, &ch->super); \ - ch->timer_active = true; \ - pmix_event_assign(&ch->ev, pmix_globals.evbase, -1, 0, \ - pmix_event_timeout_cb, ch); \ - PMIX_POST_OBJECT(ch); \ - pmix_event_add(&ch->ev, &pmix_globals.event_window); \ - } else { \ - /* add this peer to the array of sources */ \ - (void)strncpy(proc.nspace, (p)->nptr->nspace, PMIX_MAX_NSLEN); \ - proc.rank = (p)->info->pname.rank; \ - ninfo = ch->nallocated + 1; \ - PMIX_INFO_CREATE(info, ninfo); \ - /* must keep the hdlr name and return object at the end, so prepend */ \ - PMIX_INFO_LOAD(&info[0], PMIX_PROCID, \ - &proc, PMIX_PROC); \ - for (n=0; n < ch->ninfo; n++) { \ - PMIX_INFO_XFER(&info[n+1], &ch->info[n]); \ - } \ - PMIX_INFO_FREE(ch->info, ch->nallocated); \ - ch->nallocated = ninfo; \ - ch->info = info; \ - ch->ninfo = ninfo - 2; \ - /* reset the timer */ \ - pmix_event_del(&ch->ev); \ - PMIX_POST_OBJECT(ch); \ - pmix_event_add(&ch->ev, &pmix_globals.event_window); \ - } \ +#define PMIX_REPORT_EVENT(e, p, r, f) \ + do { \ + pmix_event_chain_t *ch, *cp; \ + size_t n, ninfo; \ + pmix_info_t *info; \ + pmix_proc_t proc; \ + \ + ch = NULL; \ + /* see if we already have this event cached */ \ + PMIX_LIST_FOREACH(cp, &pmix_globals.cached_events, pmix_event_chain_t) { \ + if (cp->status == (e)) { \ + ch = cp; \ + break; \ + } \ + } \ + if (NULL == ch) { \ + /* nope - need to add it */ \ + ch = PMIX_NEW(pmix_event_chain_t); \ + ch->status = (e); \ + ch->range = (r); \ + PMIX_LOAD_PROCID(&ch->source, (p)->nptr->nspace, \ + (p)->info->pname.rank); \ + PMIX_PROC_CREATE(ch->affected, 1); \ + ch->naffected = 1; \ + PMIX_LOAD_PROCID(ch->affected, (p)->nptr->nspace, \ + (p)->info->pname.rank); \ + /* if I'm a client or tool and this is my server, then we don't */ \ + /* set the targets - otherwise, we do */ \ + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && \ + !PMIX_CHECK_PROCID(&pmix_client_globals.myserver->info->pname, \ + &(p)->info->pname)) { \ + PMIX_PROC_CREATE(ch->targets, 1); \ + ch->ntargets = 1; \ + PMIX_LOAD_PROCID(ch->targets, (p)->nptr->nspace, PMIX_RANK_WILDCARD); \ + } \ + /* if this is lost-connection-to-server, then we let it go to */ \ + /* the default event handler - otherwise, we don't */ \ + if (PMIX_ERR_LOST_CONNECTION_TO_SERVER != (e) && \ + PMIX_ERR_UNREACH != (e)) { \ + ch->ninfo = 1; \ + ch->nallocated = 3; \ + PMIX_INFO_CREATE(ch->info, ch->nallocated); \ + /* mark for non-default handlers only */ \ + PMIX_INFO_LOAD(&ch->info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); \ + } else { \ + ch->nallocated = 2; \ + PMIX_INFO_CREATE(ch->info, ch->nallocated); \ + } \ + ch->final_cbfunc = (f); \ + ch->final_cbdata = ch; \ + /* cache it */ \ + pmix_list_append(&pmix_globals.cached_events, &ch->super); \ + ch->timer_active = true; \ + pmix_event_assign(&ch->ev, pmix_globals.evbase, -1, 0, \ + pmix_event_timeout_cb, ch); \ + PMIX_POST_OBJECT(ch); \ + pmix_event_add(&ch->ev, &pmix_globals.event_window); \ + } else { \ + /* add this peer to the array of sources */ \ + pmix_strncpy(proc.nspace, (p)->nptr->nspace, PMIX_MAX_NSLEN); \ + proc.rank = (p)->info->pname.rank; \ + ninfo = ch->nallocated + 1; \ + PMIX_INFO_CREATE(info, ninfo); \ + /* must keep the hdlr name and return object at the end, so prepend */ \ + PMIX_INFO_LOAD(&info[0], PMIX_PROCID, \ + &proc, PMIX_PROC); \ + for (n=0; n < ch->ninfo; n++) { \ + PMIX_INFO_XFER(&info[n+1], &ch->info[n]); \ + } \ + PMIX_INFO_FREE(ch->info, ch->nallocated); \ + ch->nallocated = ninfo; \ + ch->info = info; \ + ch->ninfo = ninfo - 2; \ + /* reset the timer */ \ + pmix_event_del(&ch->ev); \ + PMIX_POST_OBJECT(ch); \ + pmix_event_add(&ch->ev, &pmix_globals.event_window); \ + } \ } while(0) diff --git a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c index d59cfdee17..c2585ea84a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c +++ b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_notification.c @@ -1,8 +1,8 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * * $COPYRIGHT$ @@ -39,7 +39,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, const pmix_proc_t *source, pmix_data_range_t range, - pmix_info_t info[], size_t ninfo, + const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { int rc; @@ -51,18 +51,20 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, return PMIX_ERR_INIT; } - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); + pmix_output_verbose(2, pmix_server_globals.event_output, - "pmix_server_notify_event source = %s:%d event_status = %d", + "pmix_server_notify_event source = %s:%d event_status = %s", (NULL == source) ? "UNKNOWN" : source->nspace, - (NULL == source) ? PMIX_RANK_WILDCARD : source->rank, status); + (NULL == source) ? PMIX_RANK_WILDCARD : source->rank, PMIx_Error_string(status)); + rc = pmix_server_notify_client_of_event(status, source, range, info, ninfo, cbfunc, cbdata); - if (PMIX_SUCCESS != rc) { + + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { PMIX_ERROR_LOG(rc); } return rc; @@ -108,6 +110,49 @@ static void notify_event_cbfunc(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr, PMIX_RELEASE(cb); } +static pmix_status_t notify_event_cache(pmix_notify_caddy_t *cd) +{ + pmix_status_t rc; + int j; + pmix_notify_caddy_t *pk; + int idx; + time_t etime; + + /* add to our cache */ + rc = pmix_hotel_checkin(&pmix_globals.notifications, cd, &cd->room); + /* if there wasn't room, then search for the longest tenured + * occupant and evict them */ + if (PMIX_SUCCESS != rc) { + etime = 0; + idx = -1; + for (j=0; j < pmix_globals.max_events; j++) { + pmix_hotel_knock(&pmix_globals.notifications, j, (void**)&pk); + if (NULL == pk) { + /* hey, there is room! */ + pmix_hotel_checkin_with_res(&pmix_globals.notifications, cd, &cd->room); + return PMIX_SUCCESS; + } + /* check the age */ + if (0 == j) { + etime = pk->ts; + idx = j; + } else { + if (difftime(pk->ts, etime) < 0) { + etime = pk->ts; + idx = j; + } + } + } + if (0 <= idx) { + /* we found the oldest occupant - evict it */ + pmix_hotel_checkout_and_return_occupant(&pmix_globals.notifications, idx, (void**)&pk); + PMIX_RELEASE(pk); + rc = pmix_hotel_checkin(&pmix_globals.notifications, cd, &cd->room); + } + } + return rc; +} + /* as a client, we pass the notification to our server */ static pmix_status_t notify_server_of_event(pmix_status_t status, const pmix_proc_t *source, @@ -121,11 +166,13 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, pmix_cb_t *cb; pmix_event_chain_t *chain; size_t n; - pmix_notify_caddy_t *cd, *rbout; + pmix_notify_caddy_t *cd; pmix_output_verbose(2, pmix_client_globals.event_output, - "client: notifying server %s:%d of status %s for range %s", + "[%s:%d] client: notifying server %s:%d of status %s for range %s", pmix_globals.myid.nspace, pmix_globals.myid.rank, + pmix_client_globals.myserver->info->pname.nspace, + pmix_client_globals.myserver->info->pname.rank, PMIx_Error_string(status), PMIx_Data_range_string(range)); if (PMIX_RANGE_PROC_LOCAL != range) { @@ -172,7 +219,7 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, /* setup for our own local callbacks */ chain = PMIX_NEW(pmix_event_chain_t); chain->status = status; - (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); chain->source.rank = pmix_globals.myid.rank; /* we always leave space for event hdlr name and a callback object */ chain->nallocated = ninfo + 2; @@ -185,10 +232,10 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, cd = PMIX_NEW(pmix_notify_caddy_t); cd->status = status; if (NULL == source) { - (void)strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); + pmix_strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); cd->source.rank = PMIX_RANK_UNDEF; } else { - (void)strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); cd->source.rank = source->rank; } cd->range = range; @@ -200,28 +247,28 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, for (n=0; n < cd->ninfo; n++) { PMIX_INFO_XFER(&cd->info[n], &chain->info[n]); } - if (NULL != chain->targets) { - cd->ntargets = chain->ntargets; - PMIX_PROC_CREATE(cd->targets, cd->ntargets); - memcpy(cd->targets, chain->targets, cd->ntargets * sizeof(pmix_proc_t)); - } - if (NULL != chain->affected) { - cd->naffected = chain->naffected; - PMIX_PROC_CREATE(cd->affected, cd->naffected); - if (NULL == cd->affected) { - cd->naffected = 0; - rc = PMIX_ERR_NOMEM; - goto cleanup; - } - memcpy(cd->affected, chain->affected, cd->naffected * sizeof(pmix_proc_t)); - } } - - /* add to our cache */ - rbout = pmix_ring_buffer_push(&pmix_globals.notifications, cd); - /* if an older event was bumped, release it */ - if (NULL != rbout) { - PMIX_RELEASE(rbout); + if (NULL != chain->targets) { + cd->ntargets = chain->ntargets; + PMIX_PROC_CREATE(cd->targets, cd->ntargets); + memcpy(cd->targets, chain->targets, cd->ntargets * sizeof(pmix_proc_t)); + } + if (NULL != chain->affected) { + cd->naffected = chain->naffected; + PMIX_PROC_CREATE(cd->affected, cd->naffected); + if (NULL == cd->affected) { + cd->naffected = 0; + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + memcpy(cd->affected, chain->affected, cd->naffected * sizeof(pmix_proc_t)); + } + /* cache it */ + rc = notify_event_cache(cd); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(cd); + goto cleanup; } if (PMIX_RANGE_PROC_LOCAL != range && NULL != msg) { @@ -235,8 +282,10 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, cb->cbdata = cbdata; /* send to the server */ pmix_output_verbose(2, pmix_client_globals.event_output, - "client: notifying server %s:%d - sending", - pmix_globals.myid.nspace, pmix_globals.myid.rank); + "[%s:%d] client: notifying server %s:%d - sending", + pmix_globals.myid.nspace, pmix_globals.myid.rank, + pmix_client_globals.myserver->info->pname.nspace, + pmix_client_globals.myserver->info->pname.rank); PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, msg, notify_event_cbfunc, cb); if (PMIX_SUCCESS != rc) { @@ -306,9 +355,9 @@ static void progress_local_event_hdlr(pmix_status_t status, /* save this handler's returned status */ if (NULL != chain->evhdlr->name) { - (void)strncpy(newinfo[cnt].key, chain->evhdlr->name, PMIX_MAX_KEYLEN); + pmix_strncpy(newinfo[cnt].key, chain->evhdlr->name, PMIX_MAX_KEYLEN); } else { - (void)strncpy(newinfo[cnt].key, "UNKNOWN", PMIX_MAX_KEYLEN); + pmix_strncpy(newinfo[cnt].key, "UNKNOWN", PMIX_MAX_KEYLEN); } newinfo[cnt].value.type = PMIX_STATUS; newinfo[cnt].value.data.status = status; @@ -710,6 +759,8 @@ void pmix_invoke_local_event_hdlr(pmix_event_chain_t *chain) /* we still have to call their final callback */ if (NULL != chain->final_cbfunc) { chain->final_cbfunc(rc, chain->final_cbdata); + } else { + PMIX_RELEASE(chain); } return; @@ -756,17 +807,19 @@ static void local_cbfunc(pmix_status_t status, void *cbdata) static void _notify_client_event(int sd, short args, void *cbdata) { pmix_notify_caddy_t *cd = (pmix_notify_caddy_t*)cbdata; - pmix_notify_caddy_t *rbout; pmix_regevents_info_t *reginfoptr; pmix_peer_events_info_t *pr; pmix_event_chain_t *chain; - size_t n; + size_t n, nleft; bool matched, holdcd; pmix_buffer_t *bfr; pmix_cmd_t cmd = PMIX_NOTIFY_CMD; pmix_status_t rc; pmix_list_t trk; pmix_namelist_t *nm; + pmix_namespace_t *nptr, *tmp; + pmix_range_trkr_t rngtrk; + pmix_proc_t proc; /* need to acquire the object from its originating thread */ PMIX_ACQUIRE_OBJECT(cd); @@ -796,11 +849,9 @@ static void _notify_client_event(int sd, short args, void *cbdata) * the message until all local procs have received it, or it ages to * the point where it gets pushed out by more recent events */ PMIX_RETAIN(cd); - rbout = pmix_ring_buffer_push(&pmix_globals.notifications, cd); - - /* if an older event was bumped, release it */ - if (NULL != rbout) { - PMIX_RELEASE(rbout); + rc = notify_event_cache(cd); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); } } @@ -808,7 +859,7 @@ static void _notify_client_event(int sd, short args, void *cbdata) * against our registrations */ chain = PMIX_NEW(pmix_event_chain_t); chain->status = cd->status; - (void)strncpy(chain->source.nspace, cd->source.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(chain->source.nspace, cd->source.nspace, PMIX_MAX_NSLEN); chain->source.rank = cd->source.rank; /* we always leave space for a callback object and * the evhandler name. */ @@ -817,29 +868,52 @@ static void _notify_client_event(int sd, short args, void *cbdata) /* prep the chain for processing */ pmix_prep_event_chain(chain, cd->info, cd->ninfo, true); - if (0 < cd->ninfo) { - /* copy setup to the cd object */ - cd->nondefault = chain->nondefault; - if (NULL != chain->targets) { - cd->ntargets = chain->ntargets; - PMIX_PROC_CREATE(cd->targets, cd->ntargets); - memcpy(cd->targets, chain->targets, cd->ntargets * sizeof(pmix_proc_t)); - } - if (NULL != chain->affected) { - cd->naffected = chain->naffected; - PMIX_PROC_CREATE(cd->affected, cd->naffected); - if (NULL == cd->affected) { - cd->naffected = 0; - /* notify the caller */ - if (NULL != cd->cbfunc) { - cd->cbfunc(PMIX_ERR_NOMEM, cd->cbdata); + /* copy setup to the cd object */ + cd->nondefault = chain->nondefault; + if (NULL != chain->targets) { + cd->ntargets = chain->ntargets; + PMIX_PROC_CREATE(cd->targets, cd->ntargets); + memcpy(cd->targets, chain->targets, cd->ntargets * sizeof(pmix_proc_t)); + /* compute the number of targets that need to be notified */ + nleft = 0; + for (n=0; n < cd->ntargets; n++) { + /* if this is a single proc, then increment by one */ + if (PMIX_RANK_VALID >= cd->targets[n].rank) { + ++nleft; + } else { + /* look up the nspace for this proc */ + nptr = NULL; + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { + if (PMIX_CHECK_NSPACE(tmp->nspace, cd->targets[n].nspace)) { + nptr = tmp; + break; + } } - PMIX_RELEASE(cd); - PMIX_RELEASE(chain); - return; + /* if we don't yet know it, then nothing to do */ + if (NULL == nptr) { + nleft = SIZE_MAX; + break; + } + /* might notify all local members */ + nleft += nptr->nlocalprocs; } - memcpy(cd->affected, chain->affected, cd->naffected * sizeof(pmix_proc_t)); } + cd->nleft = nleft; + } + if (NULL != chain->affected) { + cd->naffected = chain->naffected; + PMIX_PROC_CREATE(cd->affected, cd->naffected); + if (NULL == cd->affected) { + cd->naffected = 0; + /* notify the caller */ + if (NULL != cd->cbfunc) { + cd->cbfunc(PMIX_ERR_NOMEM, cd->cbdata); + } + PMIX_RELEASE(cd); + PMIX_RELEASE(chain); + return; + } + memcpy(cd->affected, chain->affected, cd->naffected * sizeof(pmix_proc_t)); } /* if they provided a PMIX_EVENT_CUSTOM_RANGE info object but @@ -856,10 +930,11 @@ static void _notify_client_event(int sd, short args, void *cbdata) return; } - holdcd = false; if (PMIX_RANGE_PROC_LOCAL != cd->range) { PMIX_CONSTRUCT(&trk, pmix_list_t); + rngtrk.procs = NULL; + rngtrk.nprocs = 0; /* cycle across our registered events and send the message to * any client who registered for it */ PMIX_LIST_FOREACH(reginfoptr, &pmix_server_globals.events, pmix_regevents_info_t) { @@ -883,18 +958,33 @@ static void _notify_client_event(int sd, short args, void *cbdata) if (matched) { continue; } - /* if we were given specific targets, check if this is one */ + /* check if the affected procs (if given) match those they + * wanted to know about */ + if (!pmix_notify_check_affected(cd->affected, cd->naffected, + pr->affected, pr->naffected)) { + continue; + } + /* check the range */ + if (NULL == cd->targets) { + rngtrk.procs = &cd->source; + rngtrk.nprocs = 1; + } else { + rngtrk.procs = cd->targets; + rngtrk.nprocs = cd->ntargets; + } + rngtrk.range = cd->range; + PMIX_LOAD_PROCID(&proc, pr->peer->info->pname.nspace, pr->peer->info->pname.rank); + if (!pmix_notify_check_range(&rngtrk, &proc)) { + continue; + } if (NULL != cd->targets) { - matched = false; - for (n=0; n < cd->ntargets; n++) { - if (PMIX_CHECK_PROCID(&pr->peer->info->pname, &cd->targets[n])) { - matched = true; - break; - } - } - if (!matched) { - /* do not notify this one */ - continue; + /* track the number of targets we have left to notify */ + --cd->nleft; + /* if the event was cached and this is the last one, + * then evict this event from the cache */ + if (0 == cd->nleft) { + pmix_hotel_checkout(&pmix_globals.notifications, cd->room); + PMIX_RELEASE(cd); } } pmix_output_verbose(2, pmix_server_globals.event_output, @@ -950,15 +1040,17 @@ static void _notify_client_event(int sd, short args, void *cbdata) continue; } } - PMIX_SERVER_QUEUE_REPLY(pr->peer, 0, bfr); + PMIX_SERVER_QUEUE_REPLY(rc, pr->peer, 0, bfr); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(bfr); + } } } } PMIX_LIST_DESTRUCT(&trk); if (PMIX_RANGE_LOCAL != cd->range && PMIX_CHECK_PROCID(&cd->source, &pmix_globals.myid)) { /* if we are the source, then we need to post this upwards as - * well so the host RM can broadcast it as necessary - we rely - * on the host RM to _not_ deliver this back to us! */ + * well so the host RM can broadcast it as necessary */ if (NULL != pmix_host_server.notify_event) { /* mark that we sent it upstairs so we don't release * the caddy until we return from the host RM */ @@ -1002,23 +1094,27 @@ pmix_status_t pmix_server_notify_client_of_event(pmix_status_t status, "pmix_server: notify client of event %s", PMIx_Error_string(status)); - /* check for prior processing */ - if (NULL != info && PMIX_CHECK_KEY(&info[ninfo], PMIX_SERVER_INTERNAL_NOTIFY)) { - return PMIX_OPERATION_SUCCEEDED; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_PROXY) && + PMIX_CHECK_PROCID(info[n].value.data.proc, &pmix_globals.myid)) { + return PMIX_OPERATION_SUCCEEDED; + } + } } cd = PMIX_NEW(pmix_notify_caddy_t); cd->status = status; if (NULL == source) { - (void)strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); + pmix_strncpy(cd->source.nspace, "UNDEF", PMIX_MAX_NSLEN); cd->source.rank = PMIX_RANK_UNDEF; } else { - (void)strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->source.nspace, source->nspace, PMIX_MAX_NSLEN); cd->source.rank = source->rank; } cd->range = range; /* have to copy the info to preserve it for future when cached */ - if (0 < ninfo) { + if (0 < ninfo && NULL != info) { cd->ninfo = ninfo; PMIX_INFO_CREATE(cd->info, cd->ninfo); /* need to copy the info */ @@ -1053,37 +1149,34 @@ bool pmix_notify_check_range(pmix_range_trkr_t *rng, return true; } if (PMIX_RANGE_NAMESPACE == rng->range) { - if (0 == strncmp(pmix_globals.myid.nspace, proc->nspace, PMIX_MAX_NSLEN)) { - return true; + for (n=0; n < rng->nprocs; n++) { + if (PMIX_CHECK_NSPACE(rng->procs[n].nspace, proc->nspace)) { + return true; + } } return false; } if (PMIX_RANGE_PROC_LOCAL == rng->range) { - if (0 == strncmp(pmix_globals.myid.nspace, proc->nspace, PMIX_MAX_NSLEN) && - pmix_globals.myid.rank == proc->rank) { - return true; + for (n=0; n < rng->nprocs; n++) { + if (PMIX_CHECK_PROCID(&rng->procs[n], proc)) { + return true; + } } return false; } if (PMIX_RANGE_CUSTOM == rng->range) { - if (NULL != rng->procs) { - /* see if this proc was included */ - for (n=0; n < rng->nprocs; n++) { - if (0 != strncmp(rng->procs[n].nspace, proc->nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (PMIX_RANK_WILDCARD == rng->procs[n].rank || - rng->procs[n].rank == proc->rank) { - return true; - } + /* see if this proc was included */ + for (n=0; n < rng->nprocs; n++) { + if (0 != strncmp(rng->procs[n].nspace, proc->nspace, PMIX_MAX_NSLEN)) { + continue; + } + if (PMIX_RANK_WILDCARD == rng->procs[n].rank || + rng->procs[n].rank == proc->rank) { + return true; } - /* if we get here, then this proc isn't in range */ - return false; - } else { - /* if they didn't give us a list, then assume - * everyone included */ - return true; } + /* if we get here, then this proc isn't in range */ + return false; } /* if it is anything else, then reject it */ @@ -1106,12 +1199,7 @@ bool pmix_notify_check_affected(pmix_proc_t *interested, size_t ninterested, /* check if the two overlap */ for (n=0; n < naffected; n++) { for (m=0; m < ninterested; m++) { - if (0 != strncmp(affected[n].nspace, interested[m].nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (PMIX_RANK_WILDCARD == interested[m].rank || - PMIX_RANK_WILDCARD == affected[n].rank || - affected[n].rank == interested[m].rank) { + if (PMIX_CHECK_PROCID(&affected[n], &interested[m])) { return true; } } diff --git a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c index d1c95358ff..2607d6b101 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c +++ b/opal/mca/pmix/pmix3x/pmix/src/event/pmix_event_registration.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -95,7 +95,11 @@ static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr, PMIX_BFROPS_UNPACK(rc, peer, buf, &ret, &cnt, PMIX_STATUS); if ((PMIX_SUCCESS != rc) || (PMIX_SUCCESS != ret)) { - PMIX_ERROR_LOG(rc); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } else { + PMIX_ERROR_LOG(ret); + } /* remove the err handler and call the error handler reg completion callback fn.*/ if (NULL == rb->list) { if (NULL != rb->hdlr) { @@ -293,11 +297,7 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) PMIX_INFO_CREATE(cd2->info, cd2->ninfo); n=0; PMIX_LIST_FOREACH(ixfer, xfer, pmix_info_caddy_t) { - (void)strncpy(cd2->info[n].key, ixfer->info[n].key, PMIX_MAX_KEYLEN); - PMIX_BFROPS_VALUE_LOAD(pmix_client_globals.myserver, - &cd2->info[n].value, - &ixfer->info[n].value.data, - ixfer->info[n].value.type); + PMIX_INFO_XFER(&cd2->info[n], ixfer->info); ++n; } } @@ -333,16 +333,17 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) NULL != pmix_host_server.register_events) { pmix_output_verbose(2, pmix_client_globals.event_output, "pmix: _add_hdlr registering with server"); - if (PMIX_SUCCESS != (rc = pmix_host_server.register_events(cd->codes, cd->ncodes, - cd2->info, cd2->ninfo, - reg_cbfunc, cd2))) { + rc = pmix_host_server.register_events(cd->codes, cd->ncodes, + cd2->info, cd2->ninfo, + reg_cbfunc, cd2); + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { if (NULL != cd2->info) { PMIX_INFO_FREE(cd2->info, cd2->ninfo); } PMIX_RELEASE(cd2); return rc; } - return PMIX_ERR_WOULD_BLOCK; + return PMIX_SUCCESS; } else { if (NULL != cd2->info) { PMIX_INFO_FREE(cd2->info, cd2->ninfo); @@ -355,13 +356,15 @@ static pmix_status_t _add_hdlr(pmix_rshift_caddy_t *cd, pmix_list_t *xfer) static void check_cached_events(pmix_rshift_caddy_t *cd) { - size_t i, n; + size_t n; pmix_notify_caddy_t *ncd; bool found, matched; pmix_event_chain_t *chain; + int j; - for (i=0; i < (size_t)pmix_globals.notifications.size; i++) { - if (NULL == (ncd = (pmix_notify_caddy_t*)pmix_ring_buffer_poke(&pmix_globals.notifications, i))) { + for (j=0; j < pmix_globals.max_events; j++) { + pmix_hotel_knock(&pmix_globals.notifications, j, (void**)&ncd); + if (NULL == ncd) { continue; } found = false; @@ -381,15 +384,11 @@ static void check_cached_events(pmix_rshift_caddy_t *cd) if (!found) { continue; } - /* if we were given specific targets, check if we are one */ + /* if we were given specific targets, check if we are one */ if (NULL != ncd->targets) { matched = false; for (n=0; n < ncd->ntargets; n++) { - if (0 != strncmp(pmix_globals.myid.nspace, ncd->targets[n].nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (PMIX_RANK_WILDCARD == ncd->targets[n].rank || - pmix_globals.myid.rank == ncd->targets[n].rank) { + if (PMIX_CHECK_PROCID(&pmix_globals.myid, &ncd->targets[n])) { matched = true; break; } @@ -407,7 +406,7 @@ static void check_cached_events(pmix_rshift_caddy_t *cd) /* create the chain */ chain = PMIX_NEW(pmix_event_chain_t); chain->status = ncd->status; - (void)strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(chain->source.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); chain->source.rank = pmix_globals.myid.rank; /* we always leave space for event hdlr name and a callback object */ chain->nallocated = ncd->ninfo + 2; @@ -439,6 +438,12 @@ static void check_cached_events(pmix_rshift_caddy_t *cd) } } } + /* check this event out of the cache since we + * are processing it */ + pmix_hotel_checkout(&pmix_globals.notifications, ncd->room); + /* release the storage */ + PMIX_RELEASE(ncd); + /* we don't want this chain to propagate, so indicate it * should only be run as a single-shot */ chain->endchain = true; @@ -493,8 +498,6 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_HDLR_NAME, PMIX_MAX_KEYLEN)) { name = cd->info[n].value.data.string; - } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_ENVIRO_LEVEL, PMIX_MAX_KEYLEN)) { - cd->enviro = PMIX_INFO_TRUE(&cd->info[n]); } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { cbobject = cd->info[n].value.data.ptr; } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_HDLR_FIRST_IN_CATEGORY, PMIX_MAX_KEYLEN)) { @@ -519,17 +522,34 @@ static void reg_event_hdlr(int sd, short args, void *cbdata) } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { cd->affected = cd->info[n].value.data.proc; cd->naffected = 1; + ixfer = PMIX_NEW(pmix_info_caddy_t); + ixfer->info = &cd->info[n]; + ixfer->ninfo = 1; + pmix_list_append(&xfer, &ixfer->super); } else if (0 == strncmp(cd->info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { cd->affected = (pmix_proc_t*)cd->info[n].value.data.darray->array; cd->naffected = cd->info[n].value.data.darray->size; + ixfer = PMIX_NEW(pmix_info_caddy_t); + ixfer->info = &cd->info[n]; + ixfer->ninfo = 1; + pmix_list_append(&xfer, &ixfer->super); } else { ixfer = PMIX_NEW(pmix_info_caddy_t); ixfer->info = &cd->info[n]; + ixfer->ninfo = 1; pmix_list_append(&xfer, &ixfer->super); } } } + /* check the codes for system events */ + for (n=0; n < cd->ncodes; n++) { + if (PMIX_SYSTEM_EVENT(cd->codes[n])) { + cd->enviro = true; + break; + } + } + /* if they indicated this is to be the "first" or "last" event, then * first check to ensure they didn't already direct some * other event into the same cherished position */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h b/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h index 1e731323b7..7468985a04 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h +++ b/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc-internal.h @@ -3,7 +3,7 @@ * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -22,7 +22,7 @@ #include #if PMIX_HAVE_HWLOC -#include +#include PMIX_HWLOC_HEADER #if HWLOC_API_VERSION < 0x00010b00 #define HWLOC_OBJ_NUMANODE HWLOC_OBJ_NODE diff --git a/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c b/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c index 84050ef7bb..14dbae9076 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c +++ b/opal/mca/pmix/pmix3x/pmix/src/hwloc/hwloc.c @@ -82,7 +82,7 @@ static int set_flags(hwloc_topology_t topo, unsigned int flags) } return PMIX_SUCCESS; } -#endif +#endif // have_hwloc pmix_status_t pmix_hwloc_get_topology(pmix_info_t *info, size_t ninfo) { @@ -493,7 +493,7 @@ pmix_status_t pmix_hwloc_get_topology(pmix_info_t *info, size_t ninfo) return PMIX_SUCCESS; #else // PMIX_HAVE_HWLOC - return PMIX_ERR_NOT_SUPPORTED; + return PMIX_SUCCESS; #endif } diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/Makefile.include b/opal/mca/pmix/pmix3x/pmix/src/include/Makefile.include index 7fcbe03aeb..4ab3952e1c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/include/Makefile.include +++ b/opal/mca/pmix/pmix3x/pmix/src/include/Makefile.include @@ -10,7 +10,7 @@ # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. # Copyright (c) 2007-2016 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. @@ -41,7 +41,8 @@ headers += \ include/pmix_config_top.h \ include/pmix_config_bottom.h \ include/pmix_portable_platform.h \ - include/frameworks.h + include/frameworks.h \ + include/pmix_stdatomic.h endif ! PMIX_EMBEDDED_MODE diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_config_bottom.h b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_config_bottom.h index 8e9cf2a113..bea74d6e29 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_config_bottom.h +++ b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_config_bottom.h @@ -13,7 +13,7 @@ * Copyright (c) 2009-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -573,4 +573,5 @@ typedef PMIX_PTRDIFF_TYPE ptrdiff_t; #undef HAVE_CONFIG_H #endif /* PMIX_BUILDING */ + #endif /* PMIX_CONFIG_BOTTOM_H */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c index 8096896b24..8959ba5684 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c +++ b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c @@ -1,9 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Research Organization for Information Science - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -96,7 +96,7 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_cleanup_dir_t, pmix_list_item_t, cdcon, cddes); -static void nscon(pmix_nspace_t *p) +static void nscon(pmix_namespace_t *p) { p->nspace = NULL; p->nprocs = 0; @@ -113,7 +113,7 @@ static void nscon(pmix_nspace_t *p) PMIX_CONSTRUCT(&p->epilog.ignores, pmix_list_t); PMIX_CONSTRUCT(&p->setup_data, pmix_list_t); } -static void nsdes(pmix_nspace_t *p) +static void nsdes(pmix_namespace_t *p) { if (NULL != p->nspace) { free(p->nspace); @@ -130,7 +130,7 @@ static void nsdes(pmix_nspace_t *p) PMIX_LIST_DESTRUCT(&p->epilog.ignores); PMIX_LIST_DESTRUCT(&p->setup_data); } -PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_nspace_t, +PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_namespace_t, pmix_list_item_t, nscon, nsdes); @@ -238,6 +238,9 @@ static void iofreqdes(pmix_iof_req_t *p) if (NULL != p->peer) { PMIX_RELEASE(p->peer); } + if (NULL != p->pname.nspace) { + free(p->pname.nspace); + } } PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_iof_req_t, pmix_list_item_t, @@ -344,6 +347,8 @@ static void qdes(pmix_query_caddy_t *p) { PMIX_DESTRUCT_LOCK(&p->lock); PMIX_BYTE_OBJECT_DESTRUCT(&p->bo); + PMIX_PROC_FREE(p->targets, p->ntargets); + PMIX_INFO_FREE(p->info, p->ninfo); } PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_query_caddy_t, pmix_object_t, @@ -355,32 +360,38 @@ void pmix_execute_epilog(pmix_epilog_t *epi) pmix_cleanup_dir_t *cd, *cdnext; struct stat statbuf; int rc; + char **tmp; + size_t n; /* start with any specified files */ PMIX_LIST_FOREACH_SAFE(cf, cfnext, &epi->cleanup_files, pmix_cleanup_file_t) { /* check the effective uid/gid of the file and ensure it * matches that of the peer - we do this to provide at least * some minimum level of protection */ - rc = stat(cf->path, &statbuf); - if (0 != rc) { - pmix_output_verbose(10, pmix_globals.debug_output, - "File %s failed to stat: %d", cf->path, rc); - continue; - } - if (statbuf.st_uid != epi->uid || - statbuf.st_gid != epi->gid) { - pmix_output_verbose(10, pmix_globals.debug_output, - "File %s uid/gid doesn't match: uid %lu(%lu) gid %lu(%lu)", - cf->path, - (unsigned long)statbuf.st_uid, (unsigned long)epi->uid, - (unsigned long)statbuf.st_gid, (unsigned long)epi->gid); - continue; - } - rc = unlink(cf->path); - if (0 != rc) { - pmix_output_verbose(10, pmix_globals.debug_output, - "File %s failed to unlink: %d", cf->path, rc); + tmp = pmix_argv_split(cf->path, ','); + for (n=0; NULL != tmp[n]; n++) { + rc = stat(tmp[n], &statbuf); + if (0 != rc) { + pmix_output_verbose(10, pmix_globals.debug_output, + "File %s failed to stat: %d", tmp[n], rc); + continue; + } + if (statbuf.st_uid != epi->uid || + statbuf.st_gid != epi->gid) { + pmix_output_verbose(10, pmix_globals.debug_output, + "File %s uid/gid doesn't match: uid %lu(%lu) gid %lu(%lu)", + cf->path, + (unsigned long)statbuf.st_uid, (unsigned long)epi->uid, + (unsigned long)statbuf.st_gid, (unsigned long)epi->gid); + continue; + } + rc = unlink(tmp[n]); + if (0 != rc) { + pmix_output_verbose(10, pmix_globals.debug_output, + "File %s failed to unlink: %d", tmp[n], rc); + } } + pmix_argv_free(tmp); pmix_list_remove_item(&epi->cleanup_files, &cf->super); PMIX_RELEASE(cf); } @@ -390,27 +401,31 @@ void pmix_execute_epilog(pmix_epilog_t *epi) /* check the effective uid/gid of the file and ensure it * matches that of the peer - we do this to provide at least * some minimum level of protection */ - rc = stat(cd->path, &statbuf); - if (0 != rc) { - pmix_output_verbose(10, pmix_globals.debug_output, - "Directory %s failed to stat: %d", cd->path, rc); - continue; - } - if (statbuf.st_uid != epi->uid || - statbuf.st_gid != epi->gid) { - pmix_output_verbose(10, pmix_globals.debug_output, - "Directory %s uid/gid doesn't match: uid %lu(%lu) gid %lu(%lu)", - cd->path, - (unsigned long)statbuf.st_uid, (unsigned long)epi->uid, - (unsigned long)statbuf.st_gid, (unsigned long)epi->gid); - continue; - } - if ((statbuf.st_mode & S_IRWXU) == S_IRWXU) { - dirpath_destroy(cd->path, cd, epi); - } else { - pmix_output_verbose(10, pmix_globals.debug_output, - "Directory %s lacks permissions", cd->path); + tmp = pmix_argv_split(cd->path, ','); + for (n=0; NULL != tmp[n]; n++) { + rc = stat(tmp[n], &statbuf); + if (0 != rc) { + pmix_output_verbose(10, pmix_globals.debug_output, + "Directory %s failed to stat: %d", tmp[n], rc); + continue; + } + if (statbuf.st_uid != epi->uid || + statbuf.st_gid != epi->gid) { + pmix_output_verbose(10, pmix_globals.debug_output, + "Directory %s uid/gid doesn't match: uid %lu(%lu) gid %lu(%lu)", + cd->path, + (unsigned long)statbuf.st_uid, (unsigned long)epi->uid, + (unsigned long)statbuf.st_gid, (unsigned long)epi->gid); + continue; + } + if ((statbuf.st_mode & S_IRWXU) == S_IRWXU) { + dirpath_destroy(tmp[n], cd, epi); + } else { + pmix_output_verbose(10, pmix_globals.debug_output, + "Directory %s lacks permissions", tmp[n]); + } } + pmix_argv_free(tmp); pmix_list_remove_item(&epi->cleanup_dirs, &cd->super); PMIX_RELEASE(cd); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h index 9f565214a7..212b5b5101 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h +++ b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -36,7 +36,7 @@ #include "src/class/pmix_hash_table.h" #include "src/class/pmix_list.h" -#include "src/class/pmix_ring_buffer.h" +#include "src/class/pmix_hotel.h" #include "src/event/pmix_event.h" #include "src/threads/threads.h" @@ -178,14 +178,14 @@ typedef struct { // from this nspace pmix_list_t setup_data; // list of pmix_kval_t containing info structs having blobs // for setting up the local node for this nspace/application -} pmix_nspace_t; -PMIX_CLASS_DECLARATION(pmix_nspace_t); +} pmix_namespace_t; +PMIX_CLASS_DECLARATION(pmix_namespace_t); -/* define a caddy for quickly creating a list of pmix_nspace_t +/* define a caddy for quickly creating a list of pmix_namespace_t * objects for local, dedicated purposes */ typedef struct { pmix_list_item_t super; - pmix_nspace_t *ns; + pmix_namespace_t *ns; } pmix_nspace_caddy_t; PMIX_CLASS_DECLARATION(pmix_nspace_caddy_t); @@ -219,7 +219,7 @@ PMIX_CLASS_DECLARATION(pmix_info_caddy_t); * by the socket, not the process nspace/rank */ typedef struct pmix_peer_t { pmix_object_t super; - pmix_nspace_t *nptr; // point to the nspace object for this process + pmix_namespace_t *nptr; // point to the nspace object for this process pmix_rank_info_t *info; pmix_proc_type_t proc_type; pmix_listener_protocol_t protocol; @@ -278,6 +278,11 @@ PMIX_CLASS_DECLARATION(pmix_query_caddy_t); * - instanced in pmix_server_ops.c */ typedef struct { pmix_list_item_t super; + pmix_event_t ev; + bool event_active; + bool lost_connection; // tracker went thru lost connection procedure + bool local; // operation is strictly local + char *id; // string identifier for the collective pmix_cmd_t type; pmix_proc_t pname; bool hybrid; // true if participating procs are from more than one nspace @@ -295,6 +300,7 @@ typedef struct { pmix_collect_t collect_type; // whether or not data is to be returned at completion pmix_modex_cbfunc_t modexcbfunc; pmix_op_cbfunc_t op_cbfunc; + void *cbdata; } pmix_server_trkr_t; PMIX_CLASS_DECLARATION(pmix_server_trkr_t); @@ -340,6 +346,7 @@ PMIX_CLASS_DECLARATION(pmix_server_caddy_t); pmix_release_cbfunc_t relfn; pmix_hdlr_reg_cbfunc_t hdlrregcbfn; pmix_op_cbfunc_t opcbfn; + pmix_modex_cbfunc_t modexcbfunc; } cbfunc; void *cbdata; size_t ref; @@ -394,6 +401,11 @@ typedef struct { pmix_object_t super; pmix_event_t ev; pmix_lock_t lock; + /* timestamp receipt of the notification so we + * can evict the oldest one if we get overwhelmed */ + time_t ts; + /* what room of the hotel they are in */ + int room; pmix_status_t status; pmix_proc_t source; pmix_data_range_t range; @@ -403,6 +415,7 @@ typedef struct { */ pmix_proc_t *targets; size_t ntargets; + size_t nleft; // number of targets left to be notified /* When generating a notification, the originator can * specify the range of procs affected by this event. * For example, when creating a JOB_TERMINATED event, @@ -438,6 +451,8 @@ typedef struct { pmix_peer_t *mypeer; // my own peer object uid_t uid; // my effective uid gid_t gid; // my effective gid + char *hostname; // my hostname + uint32_t nodeid; // my nodeid, if given int pindex; pmix_event_base_t *evbase; bool external_evbase; @@ -448,7 +463,9 @@ typedef struct { struct timeval event_window; pmix_list_t cached_events; // events waiting in the window prior to processing pmix_list_t iof_requests; // list of pmix_iof_req_t IOF requests - pmix_ring_buffer_t notifications; // ring buffer of pending notifications + int max_events; // size of the notifications hotel + int event_eviction_time; // max time to cache notifications + pmix_hotel_t notifications; // hotel of pending notifications /* processes also need a place where they can store * their own internal data - e.g., data provided by * the user via the store_internal interface, as well diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_stdatomic.h b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_stdatomic.h new file mode 100644 index 0000000000..eb9562a6e6 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_stdatomic.h @@ -0,0 +1,67 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#if !defined(PMIX_STDATOMIC_H) +#define PMIX_STDATOMIC_H + +#include "pmix_stdint.h" + +#if PMIX_ASSEMBLY_BUILTIN != PMIX_BUILTIN_C11 + +typedef volatile int pmix_atomic_int_t; +typedef volatile long pmix_atomic_long_t; + +typedef volatile int32_t pmix_atomic_int32_t; +typedef volatile uint32_t pmix_atomic_uint32_t; +typedef volatile int64_t pmix_atomic_int64_t; +typedef volatile uint64_t pmix_atomic_uint64_t; + +typedef volatile size_t pmix_atomic_size_t; +typedef volatile ssize_t pmix_atomic_ssize_t; +typedef volatile intptr_t pmix_atomic_intptr_t; +typedef volatile uintptr_t pmix_atomic_uintptr_t; + +#else /* PMIX_HAVE_C__ATOMIC */ + +#include + +typedef atomic_int pmix_atomic_int_t; +typedef atomic_long pmix_atomic_long_t; + +typedef _Atomic int32_t pmix_atomic_int32_t; +typedef _Atomic uint32_t pmix_atomic_uint32_t; +typedef _Atomic int64_t pmix_atomic_int64_t; +typedef _Atomic uint64_t pmix_atomic_uint64_t; + +typedef _Atomic size_t pmix_atomic_size_t; +typedef _Atomic ssize_t pmix_atomic_ssize_t; +typedef _Atomic intptr_t pmix_atomic_intptr_t; +typedef _Atomic uintptr_t pmix_atomic_uintptr_t; + +#endif /* PMIX_HAVE_C__ATOMIC */ + +#if HAVE_PMIX_INT128_T + +/* do not use C11 atomics for __int128 if they are not lock free */ +#if PMIX_HAVE_C11_CSWAP_INT128 + +typedef _Atomic pmix_int128_t pmix_atomic_int128_t; + +#else + +typedef volatile pmix_int128_t pmix_atomic_int128_t; + +#endif + +#endif + +#endif /* !defined(PMIX_STDATOMIC_H) */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_stdint.h b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_stdint.h index 28c3099ef3..86d1cc7afe 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/include/pmix_stdint.h +++ b/opal/mca/pmix/pmix3x/pmix/src/include/pmix_stdint.h @@ -14,7 +14,7 @@ * reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,8 @@ #ifndef PMIX_STDINT_H #define PMIX_STDINT_H 1 +#include "pmix_config.h" + /* * Include what we can and define what is missing. */ @@ -125,3 +127,4 @@ typedef unsigned long long uintptr_t; #endif #endif /* PMIX_STDINT_H */ + diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_repository.c b/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_repository.c index d34e32b7b0..eb7dda21b5 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_repository.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/base/pmix_mca_base_component_repository.c @@ -15,7 +15,7 @@ * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -164,12 +164,12 @@ static int process_repository_item (const char *filename, void *data) return PMIX_ERR_OUT_OF_RESOURCE; } - /* strncpy does not guarantee a \0 */ + /* pmix_strncpy does not guarantee a \0 */ ri->ri_type[PMIX_MCA_BASE_MAX_TYPE_NAME_LEN] = '\0'; - strncpy (ri->ri_type, type, PMIX_MCA_BASE_MAX_TYPE_NAME_LEN); + pmix_strncpy (ri->ri_type, type, PMIX_MCA_BASE_MAX_TYPE_NAME_LEN); ri->ri_name[PMIX_MCA_BASE_MAX_TYPE_NAME_LEN] = '\0'; - strncpy (ri->ri_name, name, PMIX_MCA_BASE_MAX_COMPONENT_NAME_LEN); + pmix_strncpy (ri->ri_name, name, PMIX_MCA_BASE_MAX_COMPONENT_NAME_LEN); pmix_list_append (component_list, &ri->super); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h index 318f076fdc..41ee2de266 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/base.h @@ -205,6 +205,12 @@ PMIX_EXPORT extern pmix_bfrops_globals_t pmix_bfrops_globals; free(tmpbuf); \ } while (0) +/* for backwards compatibility */ +typedef struct pmix_info_array { + size_t size; + pmix_info_t *array; +} pmix_info_array_t; + /** * Internal struct used for holding registered bfrop functions @@ -268,7 +274,7 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_stub_copy_payload(struct pmix_peer_t *peer pmix_buffer_t *src); PMIX_EXPORT pmix_status_t pmix_bfrops_stub_value_xfer(struct pmix_peer_t *peer, pmix_value_t *dest, - pmix_value_t *src); + const pmix_value_t *src); PMIX_EXPORT void pmix_bfrops_stub_value_load(struct pmix_peer_t *peer, pmix_value_t *v, void *data, pmix_data_type_t type); @@ -496,9 +502,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_string(char **dest, char *src, PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_value(pmix_value_t **dest, pmix_value_t *src, pmix_data_type_t type); -PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_array(pmix_info_array_t **dest, - pmix_info_array_t *src, - pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, pmix_data_type_t type); @@ -514,9 +517,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_buf(pmix_buffer_t **dest, PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_kval(pmix_kval_t **dest, pmix_kval_t *src, pmix_data_type_t type); -PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_modex(pmix_modex_data_t **dest, - pmix_modex_data_t *src, - pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrop_base_copy_persist(pmix_persistence_t **dest, pmix_persistence_t *src, pmix_data_type_t type); @@ -538,10 +538,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_query(pmix_query_t **dest, PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_envar(pmix_envar_t **dest, pmix_envar_t *src, pmix_data_type_t type); -/**** DEPRECATED ****/ -PMIX_EXPORT pmix_status_t pmix_bfrops_base_copy_array(pmix_info_array_t **dest, - pmix_info_array_t *src, - pmix_data_type_t type); /* * "Standard" print functions @@ -596,8 +592,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_status(char **output, char *pre PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_value(char **output, char *prefix, pmix_value_t *src, pmix_data_type_t type); -PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_array(char **output, char *prefix, - pmix_info_array_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_proc(char **output, char *prefix, pmix_proc_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_app(char **output, char *prefix, @@ -608,8 +602,6 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_buf(char **output, char *prefix pmix_buffer_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_kval(char **output, char *prefix, pmix_kval_t *src, pmix_data_type_t type); -PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_modex(char **output, char *prefix, - pmix_modex_data_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_persist(char **output, char *prefix, pmix_persistence_t *src, pmix_data_type_t type); PMIX_EXPORT pmix_status_t pmix_bfrops_base_print_bo(char **output, char *prefix, @@ -678,7 +670,7 @@ PMIX_EXPORT pmix_status_t pmix_bfrops_base_value_unload(pmix_value_t *kv, size_t *sz); PMIX_EXPORT pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, - pmix_value_t *src); + const pmix_value_t *src); PMIX_EXPORT pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, pmix_value_t *p1); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c index 481eb69e76..d5bf41e94f 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_copy.c @@ -236,7 +236,7 @@ pmix_status_t pmix_bfrops_base_copy_info(pmix_info_t **dest, pmix_data_type_t type) { *dest = (pmix_info_t*)malloc(sizeof(pmix_info_t)); - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); (*dest)->flags = src->flags; return pmix_bfrops_base_value_xfer(&(*dest)->value, &src->value); } @@ -267,7 +267,7 @@ pmix_status_t pmix_bfrops_base_copy_app(pmix_app_t **dest, (*dest)->ninfo = src->ninfo; (*dest)->info = (pmix_info_t*)malloc(src->ninfo * sizeof(pmix_info_t)); for (j=0; j < src->ninfo; j++) { - (void)strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); pmix_value_xfer(&(*dest)->info[j].value, &src->info[j].value); } return PMIX_SUCCESS; @@ -300,32 +300,11 @@ pmix_status_t pmix_bfrops_base_copy_proc(pmix_proc_t **dest, if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; } - (void)strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); (*dest)->rank = src->rank; return PMIX_SUCCESS; } -pmix_status_t pmix_bfrops_base_copy_modex(pmix_modex_data_t **dest, - pmix_modex_data_t *src, - pmix_data_type_t type) -{ - *dest = (pmix_modex_data_t*)malloc(sizeof(pmix_modex_data_t)); - if (NULL == *dest) { - return PMIX_ERR_OUT_OF_RESOURCE; - } - (*dest)->blob = NULL; - (*dest)->size = 0; - if (NULL != src->blob) { - (*dest)->blob = (uint8_t*)malloc(src->size * sizeof(uint8_t)); - if (NULL == (*dest)->blob) { - return PMIX_ERR_OUT_OF_RESOURCE; - } - memcpy((*dest)->blob, src->blob, src->size * sizeof(uint8_t)); - (*dest)->size = src->size; - } - return PMIX_SUCCESS; -} - pmix_status_t pmix_bfrop_base_copy_persist(pmix_persistence_t **dest, pmix_persistence_t *src, pmix_data_type_t type) @@ -357,9 +336,9 @@ pmix_status_t pmix_bfrops_base_copy_pdata(pmix_pdata_t **dest, pmix_data_type_t type) { *dest = (pmix_pdata_t*)malloc(sizeof(pmix_pdata_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); return pmix_bfrops_base_value_xfer(&(*dest)->value, &src->value); } @@ -405,7 +384,6 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, pmix_buffer_t *pb, *sb; pmix_byte_object_t *pbo, *sbo; pmix_kval_t *pk, *sk; - pmix_modex_data_t *pm, *sm; pmix_proc_info_t *pi, *si; pmix_query_t *pq, *sq; pmix_envar_t *pe, *se; @@ -699,31 +677,6 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, } } break; - case PMIX_MODEX: - PMIX_MODEX_CREATE(p->array, src->size); - if (NULL == p->array) { - free(p); - return PMIX_ERR_NOMEM; - } - pm = (pmix_modex_data_t*)p->array; - sm = (pmix_modex_data_t*)src->array; - for (n=0; n < src->size; n++) { - memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t)); - if (NULL != sm[n].blob && 0 < sm[n].size) { - pm[n].blob = (uint8_t*)malloc(sm[n].size); - if (NULL == pm[n].blob) { - PMIX_MODEX_FREE(pm, src->size); - free(p); - return PMIX_ERR_NOMEM; - } - memcpy(pm[n].blob, sm[n].blob, sm[n].size); - pm[n].size = sm[n].size; - } else { - pm[n].blob = NULL; - pm[n].size = 0; - } - } - break; case PMIX_PERSIST: p->array = (pmix_persistence_t*)malloc(src->size * sizeof(pmix_persistence_t)); if (NULL == p->array) { @@ -876,32 +829,6 @@ pmix_status_t pmix_bfrops_base_copy_query(pmix_query_t **dest, return PMIX_SUCCESS; } -/**** DEPRECATED ****/ -pmix_status_t pmix_bfrops_base_copy_array(pmix_info_array_t **dest, - pmix_info_array_t *src, - pmix_data_type_t type) -{ - pmix_info_t *d1, *s1; - - *dest = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); - if (NULL == (*dest)) { - return PMIX_ERR_NOMEM; - } - (*dest)->size = src->size; - if (0 < src->size) { - (*dest)->array = (pmix_info_t*)malloc(src->size * sizeof(pmix_info_t)); - if (NULL == (*dest)->array) { - free(*dest); - return PMIX_ERR_NOMEM; - } - d1 = (pmix_info_t*)(*dest)->array; - s1 = (pmix_info_t*)src->array; - memcpy(d1, s1, src->size * sizeof(pmix_info_t)); - } - return PMIX_SUCCESS; -} -/*******************/ - pmix_status_t pmix_bfrops_base_copy_envar(pmix_envar_t **dest, pmix_envar_t *src, pmix_data_type_t type) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c index bcd083baf5..e93f14889a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_fns.c @@ -46,7 +46,7 @@ PMIX_EXPORT pmix_status_t pmix_value_unload(pmix_value_t *kv, } PMIX_EXPORT pmix_status_t pmix_value_xfer(pmix_value_t *dest, - pmix_value_t *src) + const pmix_value_t *src) { return pmix_bfrops_base_value_xfer(dest, src); } @@ -509,11 +509,8 @@ pmix_value_cmp_t pmix_bfrops_base_value_cmp(pmix_value_t *p, /* Xfer FUNCTIONS FOR GENERIC PMIX TYPES */ pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, - pmix_value_t *src) + const pmix_value_t *src) { - size_t n; - pmix_info_t *p1, *s1; - /* copy the right field */ p->type = src->type; switch (src->type) { @@ -643,22 +640,6 @@ pmix_status_t pmix_bfrops_base_value_xfer(pmix_value_t *p, p->data.envar.separator = src->data.envar.separator; break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - p->data.array->size = src->data.array->size; - if (0 < src->data.array->size) { - p->data.array->array = (pmix_info_t*)malloc(src->data.array->size * sizeof(pmix_info_t)); - if (NULL == p->data.array->array) { - return PMIX_ERR_NOMEM; - } - p1 = (pmix_info_t*)p->data.array->array; - s1 = (pmix_info_t*)src->data.array->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_INFO_XFER(&p1[n], &s1[n]); - } - } - break; - /********************/ default: pmix_output(0, "PMIX-XFER-VALUE: UNSUPPORTED TYPE %d", (int)src->type); return PMIX_ERROR; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c index 2e66dfe57b..4045d874ec 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_pack.c @@ -703,28 +703,6 @@ pmix_status_t pmix_bfrops_base_pack_kval(pmix_buffer_t *buffer, const void *src, return PMIX_SUCCESS; } -pmix_status_t pmix_bfrops_base_pack_modex(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ - pmix_modex_data_t *ptr; - int32_t i; - int ret; - - ptr = (pmix_modex_data_t *) src; - - for (i = 0; i < num_vals; ++i) { - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { - return ret; - } - if( 0 < ptr[i].size){ - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_byte(buffer, ptr[i].blob, ptr[i].size, PMIX_UINT8))) { - return ret; - } - } - } - return PMIX_SUCCESS; -} - pmix_status_t pmix_bfrops_base_pack_persist(pmix_buffer_t *buffer, const void *src, int32_t num_vals, pmix_data_type_t type) { @@ -1007,13 +985,6 @@ pmix_status_t pmix_bfrops_base_pack_darray(pmix_buffer_t *buffer, const void *sr } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_array(buffer, p[i].array, p[i].size, PMIX_INFO_ARRAY))) { - return ret; - } - break; - /********************/ default: pmix_output(0, "PACK-PMIX-VALUE[%s:%d]: UNSUPPORTED TYPE %d", __FILE__, __LINE__, (int)p[i].type); @@ -1236,17 +1207,10 @@ pmix_status_t pmix_bfrops_base_pack_val(pmix_buffer_t *buffer, } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_array(buffer, p->data.array, 1, PMIX_INFO_ARRAY))) { - return ret; - } - break; - /********************/ default: - pmix_output(0, "PACK-PMIX-VALUE[%s:%d]: UNSUPPORTED TYPE %d", - __FILE__, __LINE__, (int)p->type); - return PMIX_ERROR; + pmix_output(0, "PACK-PMIX-VALUE[%s:%d]: UNSUPPORTED TYPE %d", + __FILE__, __LINE__, (int)p->type); + return PMIX_ERROR; } return PMIX_SUCCESS; } @@ -1257,33 +1221,6 @@ pmix_status_t pmix_bfrops_base_pack_alloc_directive(pmix_buffer_t *buffer, const return pmix_bfrops_base_pack_byte(buffer, src, num_vals, PMIX_UINT8); } - -/**** DEPRECATED ****/ -pmix_status_t pmix_bfrops_base_pack_array(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) -{ - pmix_info_array_t *ptr; - int32_t i; - pmix_status_t ret; - - ptr = (pmix_info_array_t *) src; - - for (i = 0; i < num_vals; ++i) { - /* pack the size */ - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { - return ret; - } - if (0 < ptr[i].size) { - /* pack the values */ - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { - return ret; - } - } - } - - return PMIX_SUCCESS; -} - pmix_status_t pmix_bfrops_base_pack_iof_channel(pmix_buffer_t *buffer, const void *src, int32_t num_vals, pmix_data_type_t type) { diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c index c02fce285b..d17a731257 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_print.c @@ -1021,12 +1021,6 @@ int pmix_bfrops_base_print_status(char **output, char *prefix, src->data.envar.separator); break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - rc = asprintf(output, "%sPMIX_VALUE: Data type: INFO_ARRAY\tARRAY SIZE: %ld", - prefx, (long)src->data.array->size); - break; - /********************/ default: rc = asprintf(output, "%sPMIX_VALUE: Data type: UNKNOWN\tValue: UNPRINTABLE", prefx); break; @@ -1143,12 +1137,6 @@ int pmix_bfrops_base_print_kval(char **output, char *prefix, return PMIX_SUCCESS; } -pmix_status_t pmix_bfrops_base_print_modex(char **output, char *prefix, - pmix_modex_data_t *src, pmix_data_type_t type) -{ - return PMIX_SUCCESS; -} - int pmix_bfrops_base_print_persist(char **output, char *prefix, pmix_persistence_t *src, pmix_data_type_t type) { @@ -1702,37 +1690,3 @@ pmix_status_t pmix_bfrops_base_print_envar(char **output, char *prefix, return PMIX_SUCCESS; } } - - -/**** DEPRECATED ****/ -pmix_status_t pmix_bfrops_base_print_array(char **output, char *prefix, - pmix_info_array_t *src, pmix_data_type_t type) -{ - size_t j; - char *tmp, *tmp2, *tmp3, *pfx; - pmix_info_t *s1; - - if (0 > asprintf(&tmp, "%sARRAY SIZE: %ld", prefix, (long)src->size)) { - return PMIX_ERR_NOMEM; - } - if (0 > asprintf(&pfx, "\n%s\t", (NULL == prefix) ? "" : prefix)) { - free(tmp); - return PMIX_ERR_NOMEM; - } - s1 = (pmix_info_t*)src->array; - - for (j=0; j < src->size; j++) { - pmix_bfrops_base_print_info(&tmp2, pfx, &s1[j], PMIX_INFO); - if (0 > asprintf(&tmp3, "%s%s", tmp, tmp2)) { - free(tmp); - free(tmp2); - return PMIX_ERR_NOMEM; - } - free(tmp); - free(tmp2); - tmp = tmp3; - } - *output = tmp; - return PMIX_SUCCESS; -} -/********************/ diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c index afd685b437..051c35d82a 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c @@ -764,21 +764,9 @@ pmix_status_t pmix_bfrops_base_unpack_val(pmix_buffer_t *buffer, return ret; } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - /* this field is now a pointer, so we must allocate storage for it */ - val->data.array = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); - if (NULL == val->data.array) { - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_array(buffer, val->data.array, &m, PMIX_INFO_ARRAY))) { - return ret; - } - break; - /********************/ default: - pmix_output(0, "UNPACK-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)val->type); - return PMIX_ERROR; + pmix_output(0, "UNPACK-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)val->type); + return PMIX_ERROR; } return PMIX_SUCCESS; @@ -836,7 +824,7 @@ pmix_status_t pmix_bfrops_base_unpack_info(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack the directives */ m=1; @@ -890,7 +878,7 @@ pmix_status_t pmix_bfrops_base_unpack_pdata(pmix_buffer_t *buffer, void *dest, PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to @@ -982,7 +970,7 @@ pmix_status_t pmix_bfrops_base_unpack_proc(pmix_buffer_t *buffer, void *dest, PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } - (void)strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); + pmix_strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); free(tmp); /* unpack the rank */ m=1; @@ -1111,38 +1099,6 @@ pmix_status_t pmix_bfrops_base_unpack_kval(pmix_buffer_t *buffer, void *dest, return PMIX_SUCCESS; } -pmix_status_t pmix_bfrops_base_unpack_modex(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ - pmix_modex_data_t *ptr; - int32_t i, n, m; - pmix_status_t ret; - - pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, - "pmix_bfrop_unpack: %d modex", *num_vals); - - ptr = (pmix_modex_data_t *) dest; - n = *num_vals; - - for (i = 0; i < n; ++i) { - memset(&ptr[i], 0, sizeof(pmix_modex_data_t)); - /* unpack the number of bytes */ - m=1; - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { - return ret; - } - if (0 < ptr[i].size) { - ptr[i].blob = (uint8_t*)malloc(ptr[i].size * sizeof(uint8_t)); - m=ptr[i].size; - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_byte(buffer, ptr[i].blob, &m, PMIX_UINT8))) { - return ret; - } - } - } - return PMIX_SUCCESS; -} - - pmix_status_t pmix_bfrops_base_unpack_persist(pmix_buffer_t *buffer, void *dest, int32_t *num_vals, pmix_data_type_t type) { @@ -1539,17 +1495,6 @@ pmix_status_t pmix_bfrops_base_unpack_darray(pmix_buffer_t *buffer, void *dest, return ret; } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - ptr[i].array = (pmix_info_array_t*)malloc(m * sizeof(pmix_info_array_t)); - if (NULL == ptr[i].array) { - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_array(buffer, ptr[i].array, &m, ptr[i].type))) { - return ret; - } - break; - /********************/ default: return PMIX_ERR_NOT_SUPPORTED; } @@ -1657,37 +1602,3 @@ pmix_status_t pmix_bfrops_base_unpack_envar(pmix_buffer_t *buffer, void *dest, } return PMIX_SUCCESS; } - -/**** DEPRECATED ****/ -pmix_status_t pmix_bfrops_base_unpack_array(pmix_buffer_t *buffer, void *dest, - int32_t *num_vals, pmix_data_type_t type) -{ - pmix_info_array_t *ptr; - int32_t i, n, m; - pmix_status_t ret; - - pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, - "pmix_bfrop_unpack: %d info arrays", *num_vals); - - ptr = (pmix_info_array_t*) dest; - n = *num_vals; - - for (i = 0; i < n; ++i) { - pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, - "pmix_bfrop_unpack: init array[%d]", i); - memset(&ptr[i], 0, sizeof(pmix_info_array_t)); - /* unpack the size of this array */ - m=1; - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { - return ret; - } - if (0 < ptr[i].size) { - ptr[i].array = (pmix_info_t*)malloc(ptr[i].size * sizeof(pmix_info_t)); - m=ptr[i].size; - if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_value(buffer, ptr[i].array, &m, PMIX_INFO))) { - return ret; - } - } - } - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h index bfb4013b87..258b727afe 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/bfrops.h @@ -324,7 +324,7 @@ typedef pmix_status_t (*pmix_bfrop_print_fn_t)(char **output, char *prefix, * @retval PMIX_ERROR(s) An appropriate error code */ typedef pmix_status_t (*pmix_bfrop_value_xfer_fn_t)(pmix_value_t *dest, - pmix_value_t *src); + const pmix_value_t *src); /** diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/copy.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/copy.c index d833e4bdc8..c4eeeb14bb 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/copy.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/copy.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -225,10 +225,8 @@ pmix_value_cmp_t pmix12_bfrop_value_cmp(pmix_value_t *p, pmix_value_t *p1) return PMIX_VALUE1_GREATER; } /* COPY FUNCTIONS FOR GENERIC PMIX TYPES */ -pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src) +pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, const pmix_value_t *src) { - pmix_info_t *p1, *s1; - /* copy the right field */ p->type = src->type; switch (src->type) { @@ -300,22 +298,7 @@ pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src) p->data.tv.tv_usec = src->data.tv.tv_usec; break; case PMIX_INFO_ARRAY: - p->data.array = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); - if (NULL == p->data.array) { - return PMIX_ERR_NOMEM; - } - p->data.array->size = src->data.array->size; - if (0 < src->data.array->size) { - p->data.array->array = (pmix_info_t*)malloc(src->data.array->size * sizeof(pmix_info_t)); - if (NULL == p->data.array->array) { - free(p->data.array); - return PMIX_ERR_NOMEM; - } - p1 = (pmix_info_t*)p->data.array->array; - s1 = (pmix_info_t*)src->data.array->array; - memcpy(p1, s1, src->data.array->size * sizeof(pmix_info_t)); - } - break; + return PMIX_ERR_NOT_SUPPORTED; case PMIX_BYTE_OBJECT: if (NULL != src->data.bo.bytes && 0 < src->data.bo.size) { p->data.bo.bytes = malloc(src->data.bo.size); @@ -356,7 +339,7 @@ pmix_status_t pmix12_bfrop_copy_info(pmix_info_t **dest, pmix_info_t *src, pmix_data_type_t type) { *dest = (pmix_info_t*)malloc(sizeof(pmix_info_t)); - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); return pmix_value_xfer(&(*dest)->value, &src->value); } @@ -381,7 +364,7 @@ pmix_status_t pmix12_bfrop_copy_app(pmix_app_t **dest, pmix_app_t *src, (*dest)->ninfo = src->ninfo; (*dest)->info = (pmix_info_t*)malloc(src->ninfo * sizeof(pmix_info_t)); for (j=0; j < src->ninfo; j++) { - (void)strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); pmix_value_xfer(&(*dest)->info[j].value, &src->info[j].value); } return PMIX_SUCCESS; @@ -427,7 +410,7 @@ pmix_status_t pmix12_bfrop_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; } - (void)strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); (*dest)->rank = src->rank; return PMIX_SUCCESS; } @@ -484,9 +467,9 @@ pmix_status_t pmix12_bfrop_copy_pdata(pmix_pdata_t **dest, pmix_data_type_t type) { *dest = (pmix_pdata_t*)malloc(sizeof(pmix_pdata_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); return pmix_value_xfer(&(*dest)->value, &src->value); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/internal.h b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/internal.h index 78dbf47666..9c74c85568 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/internal.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/internal.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -41,6 +41,55 @@ BEGIN_C_DECLS +/* DEPRECATED data type values */ +#define PMIX_MODEX 29 +#define PMIX_INFO_ARRAY 44 + +/**** PMIX MODEX STRUCT - DEPRECATED ****/ +typedef struct pmix_modex_data { + char nspace[PMIX_MAX_NSLEN+1]; + int rank; + uint8_t *blob; + size_t size; +} pmix_modex_data_t; +/* utility macros for working with pmix_modex_t structs */ +#define PMIX_MODEX_CREATE(m, n) \ + do { \ + (m) = (pmix_modex_data_t*)calloc((n) , sizeof(pmix_modex_data_t)); \ + } while (0) + +#define PMIX_MODEX_RELEASE(m) \ + do { \ + PMIX_MODEX_DESTRUCT((m)); \ + free((m)); \ + (m) = NULL; \ + } while (0) + +#define PMIX_MODEX_CONSTRUCT(m) \ + do { \ + memset((m), 0, sizeof(pmix_modex_data_t)); \ + } while (0) + +#define PMIX_MODEX_DESTRUCT(m) \ + do { \ + if (NULL != (m)->blob) { \ + free((m)->blob); \ + (m)->blob = NULL; \ + } \ + } while (0) + +#define PMIX_MODEX_FREE(m, n) \ + do { \ + size_t _s; \ + if (NULL != (m)) { \ + for (_s=0; _s < (n); _s++) { \ + PMIX_MODEX_DESTRUCT(&((m)[_s])); \ + } \ + free((m)); \ + (m) = NULL; \ + } \ + } while (0) + /* * Implementations of API functions */ @@ -58,7 +107,7 @@ pmix_status_t pmix12_bfrop_print(char **output, char *prefix, void *src, pmix_da pmix_status_t pmix12_bfrop_copy_payload(pmix_buffer_t *dest, pmix_buffer_t *src); -pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src); +pmix_status_t pmix12_bfrop_value_xfer(pmix_value_t *p, const pmix_value_t *src); void pmix12_bfrop_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/pack.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/pack.c index 07f9a74a27..efddd287de 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/pack.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/pack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2013 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -537,11 +537,6 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer, return ret; } break; - case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix12_bfrop_pack_buffer(buffer, &p->data.array, 1, PMIX_INFO_ARRAY))) { - return ret; - } - break; case PMIX_BYTE_OBJECT: if (PMIX_SUCCESS != (ret = pmix12_bfrop_pack_buffer(buffer, &p->data.bo, 1, PMIX_BYTE_OBJECT))) { return ret; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/unpack.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/unpack.c index 5604637f1b..a001728ef0 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/unpack.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v12/unpack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -670,7 +670,13 @@ static pmix_status_t unpack_val(pmix_buffer_t *buffer, pmix_value_t *val) } break; case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix12_bfrop_unpack_buffer(buffer, &val->data.array, &m, PMIX_INFO_ARRAY))) { + /* we don't know anything about info array's so we + * have to convert this to a data array */ + val->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); + val->data.darray->type = PMIX_INFO_ARRAY; + val->data.darray->size = m; + /* unpack into it */ + if (PMIX_SUCCESS != (ret = pmix12_bfrop_unpack_buffer(buffer, &val->data.darray->array, &m, PMIX_INFO_ARRAY))) { return ret; } break; @@ -743,7 +749,7 @@ pmix_status_t pmix12_bfrop_unpack_info(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to @@ -797,7 +803,7 @@ pmix_status_t pmix12_bfrop_unpack_pdata(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to @@ -881,7 +887,7 @@ pmix_status_t pmix12_bfrop_unpack_proc(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); + pmix_strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); free(tmp); /* unpack the rank */ m=1; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/copy.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/copy.c index 53fbf9b264..06720cf32e 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/copy.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/copy.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -337,7 +337,7 @@ bool pmix_value_cmp(pmix_value_t *p, pmix_value_t *p1) /* COPY FUNCTIONS FOR GENERIC PMIX TYPES - we * are not allocating memory and so we cannot * use the regular copy functions */ -pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src) +pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, const pmix_value_t *src) { size_t n, m; pmix_status_t rc; @@ -356,518 +356,506 @@ pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src) /* copy the right field */ p->type = src->type; switch (src->type) { - case PMIX_UNDEF: - break; - case PMIX_BOOL: - p->data.flag = src->data.flag; - break; - case PMIX_BYTE: - p->data.byte = src->data.byte; - break; - case PMIX_STRING: - if (NULL != src->data.string) { - p->data.string = strdup(src->data.string); - } else { - p->data.string = NULL; - } - break; - case PMIX_SIZE: - p->data.size = src->data.size; - break; - case PMIX_PID: - p->data.pid = src->data.pid; - break; - case PMIX_INT: - /* to avoid alignment issues */ - memcpy(&p->data.integer, &src->data.integer, sizeof(int)); - break; - case PMIX_INT8: - p->data.int8 = src->data.int8; - break; - case PMIX_INT16: - /* to avoid alignment issues */ - memcpy(&p->data.int16, &src->data.int16, 2); - break; - case PMIX_INT32: - /* to avoid alignment issues */ - memcpy(&p->data.int32, &src->data.int32, 4); - break; - case PMIX_INT64: - /* to avoid alignment issues */ - memcpy(&p->data.int64, &src->data.int64, 8); - break; - case PMIX_UINT: - /* to avoid alignment issues */ - memcpy(&p->data.uint, &src->data.uint, sizeof(unsigned int)); - break; - case PMIX_UINT8: - p->data.uint8 = src->data.uint8; - break; - case PMIX_UINT16: - /* to avoid alignment issues */ - memcpy(&p->data.uint16, &src->data.uint16, 2); - break; - case PMIX_UINT32: - /* to avoid alignment issues */ - memcpy(&p->data.uint32, &src->data.uint32, 4); - break; - case PMIX_UINT64: - /* to avoid alignment issues */ - memcpy(&p->data.uint64, &src->data.uint64, 8); - break; - case PMIX_FLOAT: - p->data.fval = src->data.fval; - break; - case PMIX_DOUBLE: - p->data.dval = src->data.dval; - break; - case PMIX_TIMEVAL: - memcpy(&p->data.tv, &src->data.tv, sizeof(struct timeval)); - break; - case PMIX_TIME: - memcpy(&p->data.time, &src->data.time, sizeof(time_t)); - break; - case PMIX_STATUS: - memcpy(&p->data.status, &src->data.status, sizeof(pmix_status_t)); - break; - case PMIX_PROC: - memcpy(&p->data.proc, &src->data.proc, sizeof(pmix_proc_t)); - break; - case PMIX_PROC_RANK: - memcpy(&p->data.proc, &src->data.rank, sizeof(pmix_rank_t)); - break; - case PMIX_BYTE_OBJECT: - case PMIX_COMPRESSED_STRING: - memset(&p->data.bo, 0, sizeof(pmix_byte_object_t)); - if (NULL != src->data.bo.bytes && 0 < src->data.bo.size) { - p->data.bo.bytes = malloc(src->data.bo.size); - memcpy(p->data.bo.bytes, src->data.bo.bytes, src->data.bo.size); - p->data.bo.size = src->data.bo.size; - } else { - p->data.bo.bytes = NULL; - p->data.bo.size = 0; - } - break; - case PMIX_PERSIST: - memcpy(&p->data.persist, &src->data.persist, sizeof(pmix_persistence_t)); - break; - case PMIX_SCOPE: - memcpy(&p->data.scope, &src->data.scope, sizeof(pmix_scope_t)); - break; - case PMIX_DATA_RANGE: - memcpy(&p->data.range, &src->data.range, sizeof(pmix_data_range_t)); - break; - case PMIX_PROC_STATE: - memcpy(&p->data.state, &src->data.state, sizeof(pmix_proc_state_t)); - break; - case PMIX_PROC_INFO: - PMIX_PROC_INFO_CREATE(p->data.pinfo, 1); - if (NULL != src->data.pinfo->hostname) { - p->data.pinfo->hostname = strdup(src->data.pinfo->hostname); - } - if (NULL != src->data.pinfo->executable_name) { - p->data.pinfo->executable_name = strdup(src->data.pinfo->executable_name); - } - memcpy(&p->data.pinfo->pid, &src->data.pinfo->pid, sizeof(pid_t)); - memcpy(&p->data.pinfo->exit_code, &src->data.pinfo->exit_code, sizeof(int)); - memcpy(&p->data.pinfo->state, &src->data.pinfo->state, sizeof(pmix_proc_state_t)); - break; - case PMIX_DATA_ARRAY: - p->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); - p->data.darray->type = src->data.darray->type; - p->data.darray->size = src->data.darray->size; - if (0 == p->data.darray->size || NULL == src->data.darray->array) { - p->data.darray->array = NULL; - p->data.darray->size = 0; + case PMIX_UNDEF: break; - } - /* allocate space and do the copy */ - switch (src->data.darray->type) { - case PMIX_UINT8: - case PMIX_INT8: - case PMIX_BYTE: - p->data.darray->array = (char*)malloc(src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size); + case PMIX_BOOL: + p->data.flag = src->data.flag; + break; + case PMIX_BYTE: + p->data.byte = src->data.byte; + break; + case PMIX_STRING: + if (NULL != src->data.string) { + p->data.string = strdup(src->data.string); + } else { + p->data.string = NULL; + } + break; + case PMIX_SIZE: + p->data.size = src->data.size; + break; + case PMIX_PID: + p->data.pid = src->data.pid; + break; + case PMIX_INT: + /* to avoid alignment issues */ + memcpy(&p->data.integer, &src->data.integer, sizeof(int)); + break; + case PMIX_INT8: + p->data.int8 = src->data.int8; + break; + case PMIX_INT16: + /* to avoid alignment issues */ + memcpy(&p->data.int16, &src->data.int16, 2); + break; + case PMIX_INT32: + /* to avoid alignment issues */ + memcpy(&p->data.int32, &src->data.int32, 4); + break; + case PMIX_INT64: + /* to avoid alignment issues */ + memcpy(&p->data.int64, &src->data.int64, 8); + break; + case PMIX_UINT: + /* to avoid alignment issues */ + memcpy(&p->data.uint, &src->data.uint, sizeof(unsigned int)); + break; + case PMIX_UINT8: + p->data.uint8 = src->data.uint8; + break; + case PMIX_UINT16: + /* to avoid alignment issues */ + memcpy(&p->data.uint16, &src->data.uint16, 2); + break; + case PMIX_UINT32: + /* to avoid alignment issues */ + memcpy(&p->data.uint32, &src->data.uint32, 4); + break; + case PMIX_UINT64: + /* to avoid alignment issues */ + memcpy(&p->data.uint64, &src->data.uint64, 8); + break; + case PMIX_FLOAT: + p->data.fval = src->data.fval; + break; + case PMIX_DOUBLE: + p->data.dval = src->data.dval; + break; + case PMIX_TIMEVAL: + memcpy(&p->data.tv, &src->data.tv, sizeof(struct timeval)); + break; + case PMIX_TIME: + memcpy(&p->data.time, &src->data.time, sizeof(time_t)); + break; + case PMIX_STATUS: + memcpy(&p->data.status, &src->data.status, sizeof(pmix_status_t)); + break; + case PMIX_PROC: + memcpy(&p->data.proc, &src->data.proc, sizeof(pmix_proc_t)); + break; + case PMIX_PROC_RANK: + memcpy(&p->data.proc, &src->data.rank, sizeof(pmix_rank_t)); + break; + case PMIX_BYTE_OBJECT: + case PMIX_COMPRESSED_STRING: + memset(&p->data.bo, 0, sizeof(pmix_byte_object_t)); + if (NULL != src->data.bo.bytes && 0 < src->data.bo.size) { + p->data.bo.bytes = malloc(src->data.bo.size); + memcpy(p->data.bo.bytes, src->data.bo.bytes, src->data.bo.size); + p->data.bo.size = src->data.bo.size; + } else { + p->data.bo.bytes = NULL; + p->data.bo.size = 0; + } + break; + case PMIX_PERSIST: + memcpy(&p->data.persist, &src->data.persist, sizeof(pmix_persistence_t)); + break; + case PMIX_SCOPE: + memcpy(&p->data.scope, &src->data.scope, sizeof(pmix_scope_t)); + break; + case PMIX_DATA_RANGE: + memcpy(&p->data.range, &src->data.range, sizeof(pmix_data_range_t)); + break; + case PMIX_PROC_STATE: + memcpy(&p->data.state, &src->data.state, sizeof(pmix_proc_state_t)); + break; + case PMIX_PROC_INFO: + PMIX_PROC_INFO_CREATE(p->data.pinfo, 1); + if (NULL != src->data.pinfo->hostname) { + p->data.pinfo->hostname = strdup(src->data.pinfo->hostname); + } + if (NULL != src->data.pinfo->executable_name) { + p->data.pinfo->executable_name = strdup(src->data.pinfo->executable_name); + } + memcpy(&p->data.pinfo->pid, &src->data.pinfo->pid, sizeof(pid_t)); + memcpy(&p->data.pinfo->exit_code, &src->data.pinfo->exit_code, sizeof(int)); + memcpy(&p->data.pinfo->state, &src->data.pinfo->state, sizeof(pmix_proc_state_t)); + break; + case PMIX_DATA_ARRAY: + p->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); + p->data.darray->type = src->data.darray->type; + p->data.darray->size = src->data.darray->size; + if (0 == p->data.darray->size || NULL == src->data.darray->array) { + p->data.darray->array = NULL; + p->data.darray->size = 0; break; - case PMIX_UINT16: - case PMIX_INT16: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint16_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint16_t)); - break; - case PMIX_UINT32: - case PMIX_INT32: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint32_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint32_t)); - break; - case PMIX_UINT64: - case PMIX_INT64: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint64_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint64_t)); - break; - case PMIX_BOOL: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(bool)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(bool)); - break; - case PMIX_SIZE: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(size_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(size_t)); - break; - case PMIX_PID: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(pid_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pid_t)); - break; - case PMIX_STRING: - p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - prarray = (char**)p->data.darray->array; - strarray = (char**)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != strarray[n]) { - prarray[n] = strdup(strarray[n]); + } + /* allocate space and do the copy */ + switch (src->data.darray->type) { + case PMIX_UINT8: + case PMIX_INT8: + case PMIX_BYTE: + p->data.darray->array = (char*)malloc(src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - } - break; - case PMIX_INT: - case PMIX_UINT: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(int)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(int)); - break; - case PMIX_FLOAT: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(float)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(float)); - break; - case PMIX_DOUBLE: - p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(double)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(double)); - break; - case PMIX_TIMEVAL: - p->data.darray->array = (struct timeval*)malloc(src->data.darray->size * sizeof(struct timeval)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(struct timeval)); - break; - case PMIX_TIME: - p->data.darray->array = (time_t*)malloc(src->data.darray->size * sizeof(time_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(time_t)); - break; - case PMIX_STATUS: - p->data.darray->array = (pmix_status_t*)malloc(src->data.darray->size * sizeof(pmix_status_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_status_t)); - break; - case PMIX_VALUE: - PMIX_VALUE_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pv = (pmix_value_t*)p->data.darray->array; - sv = (pmix_value_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (PMIX_SUCCESS != (rc = pmix20_bfrop_value_xfer(&pv[n], &sv[n]))) { - PMIX_VALUE_FREE(pv, src->data.darray->size); - return rc; + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size); + break; + case PMIX_UINT16: + case PMIX_INT16: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint16_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - } - break; - case PMIX_PROC: - PMIX_PROC_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_proc_t)); - break; - case PMIX_APP: - PMIX_APP_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pa = (pmix_app_t*)p->data.darray->array; - sa = (pmix_app_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sa[n].cmd) { - pa[n].cmd = strdup(sa[n].cmd); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint16_t)); + break; + case PMIX_UINT32: + case PMIX_INT32: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint32_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sa[n].argv) { - pa[n].argv = pmix_argv_copy(sa[n].argv); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint32_t)); + break; + case PMIX_UINT64: + case PMIX_INT64: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(uint64_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sa[n].env) { - pa[n].env = pmix_argv_copy(sa[n].env); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(uint64_t)); + break; + case PMIX_BOOL: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(bool)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sa[n].cwd) { - pa[n].cwd = strdup(sa[n].cwd); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(bool)); + break; + case PMIX_SIZE: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(size_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - pa[n].maxprocs = sa[n].maxprocs; - if (0 < sa[n].ninfo && NULL != sa[n].info) { - PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo); - if (NULL == pa[n].info) { - PMIX_APP_FREE(pa, src->data.darray->size); - return PMIX_ERR_NOMEM; - } - pa[n].ninfo = sa[n].ninfo; - for (m=0; m < pa[n].ninfo; m++) { - PMIX_INFO_XFER(&pa[n].info[m], &sa[n].info[m]); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(size_t)); + break; + case PMIX_PID: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(pid_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pid_t)); + break; + case PMIX_STRING: + p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + prarray = (char**)p->data.darray->array; + strarray = (char**)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != strarray[n]) { + prarray[n] = strdup(strarray[n]); } } - } - break; - case PMIX_INFO: - PMIX_INFO_CREATE(p->data.darray->array, src->data.darray->size); - p1 = (pmix_info_t*)p->data.darray->array; - s1 = (pmix_info_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type); - } - break; - case PMIX_PDATA: - PMIX_PDATA_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pd = (pmix_pdata_t*)p->data.darray->array; - sd = (pmix_pdata_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_PDATA_LOAD(&pd[n], &sd[n].proc, sd[n].key, &sd[n].value.data.flag, sd[n].value.type); - } - break; - case PMIX_BUFFER: - p->data.darray->array = (pmix_buffer_t*)malloc(src->data.darray->size * sizeof(pmix_buffer_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pb = (pmix_buffer_t*)p->data.darray->array; - sb = (pmix_buffer_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_CONSTRUCT(&pb[n], pmix_buffer_t); - pmix20_bfrop_copy_payload(&pb[n], &sb[n]); - } - break; - case PMIX_BYTE_OBJECT: - case PMIX_COMPRESSED_STRING: - p->data.darray->array = (pmix_byte_object_t*)malloc(src->data.darray->size * sizeof(pmix_byte_object_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pbo = (pmix_byte_object_t*)p->data.darray->array; - sbo = (pmix_byte_object_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sbo[n].bytes && 0 < sbo[n].size) { - pbo[n].size = sbo[n].size; - pbo[n].bytes = (char*)malloc(pbo[n].size); - memcpy(pbo[n].bytes, sbo[n].bytes, pbo[n].size); - } else { - pbo[n].bytes = NULL; - pbo[n].size = 0; + break; + case PMIX_INT: + case PMIX_UINT: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(int)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - } - break; - case PMIX_KVAL: - p->data.darray->array = (pmix_kval_t*)calloc(src->data.darray->size , sizeof(pmix_kval_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pk = (pmix_kval_t*)p->data.darray->array; - sk = (pmix_kval_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sk[n].key) { - pk[n].key = strdup(sk[n].key); + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(int)); + break; + case PMIX_FLOAT: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(float)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; } - if (NULL != sk[n].value) { - PMIX_VALUE_CREATE(pk[n].value, 1); - if (NULL == pk[n].value) { - free(p->data.darray->array); - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (rc = pmix20_bfrop_value_xfer(pk[n].value, sk[n].value))) { + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(float)); + break; + case PMIX_DOUBLE: + p->data.darray->array = (char*)malloc(src->data.darray->size * sizeof(double)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(double)); + break; + case PMIX_TIMEVAL: + p->data.darray->array = (struct timeval*)malloc(src->data.darray->size * sizeof(struct timeval)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(struct timeval)); + break; + case PMIX_TIME: + p->data.darray->array = (time_t*)malloc(src->data.darray->size * sizeof(time_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(time_t)); + break; + case PMIX_STATUS: + p->data.darray->array = (pmix_status_t*)malloc(src->data.darray->size * sizeof(pmix_status_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_status_t)); + break; + case PMIX_VALUE: + PMIX_VALUE_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pv = (pmix_value_t*)p->data.darray->array; + sv = (pmix_value_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (PMIX_SUCCESS != (rc = pmix20_bfrop_value_xfer(&pv[n], &sv[n]))) { + PMIX_VALUE_FREE(pv, src->data.darray->size); return rc; } } - } - break; - case PMIX_MODEX: - PMIX_MODEX_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pm = (pmix_modex_data_t*)p->data.darray->array; - sm = (pmix_modex_data_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t)); - if (NULL != sm[n].blob && 0 < sm[n].size) { - pm[n].blob = (uint8_t*)malloc(sm[n].size); - if (NULL == pm[n].blob) { - return PMIX_ERR_NOMEM; + break; + case PMIX_PROC: + PMIX_PROC_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_proc_t)); + break; + case PMIX_APP: + PMIX_APP_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pa = (pmix_app_t*)p->data.darray->array; + sa = (pmix_app_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != sa[n].cmd) { + pa[n].cmd = strdup(sa[n].cmd); } - memcpy(pm[n].blob, sm[n].blob, sm[n].size); - pm[n].size = sm[n].size; - } else { - pm[n].blob = NULL; - pm[n].size = 0; - } - } - break; - case PMIX_PERSIST: - p->data.darray->array = (pmix_persistence_t*)malloc(src->data.darray->size * sizeof(pmix_persistence_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_persistence_t)); - break; - case PMIX_POINTER: - p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - prarray = (char**)p->data.darray->array; - strarray = (char**)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - prarray[n] = strarray[n]; - } - break; - case PMIX_SCOPE: - p->data.darray->array = (pmix_scope_t*)malloc(src->data.darray->size * sizeof(pmix_scope_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_scope_t)); - break; - case PMIX_DATA_RANGE: - p->data.darray->array = (pmix_data_range_t*)malloc(src->data.darray->size * sizeof(pmix_data_range_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_data_range_t)); - break; - case PMIX_COMMAND: - p->data.darray->array = (pmix_cmd_t*)malloc(src->data.darray->size * sizeof(pmix_cmd_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_cmd_t)); - break; - case PMIX_INFO_DIRECTIVES: - p->data.darray->array = (pmix_info_directives_t*)malloc(src->data.darray->size * sizeof(pmix_info_directives_t)); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_info_directives_t)); - break; - case PMIX_PROC_INFO: - PMIX_PROC_INFO_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pi = (pmix_proc_info_t*)p->data.darray->array; - si = (pmix_proc_info_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - memcpy(&pi[n].proc, &si[n].proc, sizeof(pmix_proc_t)); - if (NULL != si[n].hostname) { - pi[n].hostname = strdup(si[n].hostname); - } else { - pi[n].hostname = NULL; - } - if (NULL != si[n].executable_name) { - pi[n].executable_name = strdup(si[n].executable_name); - } else { - pi[n].executable_name = NULL; - } - pi[n].pid = si[n].pid; - pi[n].exit_code = si[n].exit_code; - pi[n].state = si[n].state; - } - break; - case PMIX_DATA_ARRAY: - return PMIX_ERR_NOT_SUPPORTED; // don't support iterative arrays - case PMIX_QUERY: - PMIX_QUERY_CREATE(p->data.darray->array, src->data.darray->size); - if (NULL == p->data.darray->array) { - return PMIX_ERR_NOMEM; - } - pq = (pmix_query_t*)p->data.darray->array; - sq = (pmix_query_t*)src->data.darray->array; - for (n=0; n < src->data.darray->size; n++) { - if (NULL != sq[n].keys) { - pq[n].keys = pmix_argv_copy(sq[n].keys); - } - if (NULL != sq[n].qualifiers && 0 < sq[n].nqual) { - PMIX_INFO_CREATE(pq[n].qualifiers, sq[n].nqual); - if (NULL == pq[n].qualifiers) { - PMIX_QUERY_FREE(pq, src->data.darray->size); - return PMIX_ERR_NOMEM; + if (NULL != sa[n].argv) { + pa[n].argv = pmix_argv_copy(sa[n].argv); } - for (m=0; m < sq[n].nqual; m++) { - PMIX_INFO_XFER(&pq[n].qualifiers[m], &sq[n].qualifiers[m]); + if (NULL != sa[n].env) { + pa[n].env = pmix_argv_copy(sa[n].env); + } + if (NULL != sa[n].cwd) { + pa[n].cwd = strdup(sa[n].cwd); + } + pa[n].maxprocs = sa[n].maxprocs; + if (0 < sa[n].ninfo && NULL != sa[n].info) { + PMIX_INFO_CREATE(pa[n].info, sa[n].ninfo); + if (NULL == pa[n].info) { + PMIX_APP_FREE(pa, src->data.darray->size); + return PMIX_ERR_NOMEM; + } + pa[n].ninfo = sa[n].ninfo; + for (m=0; m < pa[n].ninfo; m++) { + PMIX_INFO_XFER(&pa[n].info[m], &sa[n].info[m]); + } } - pq[n].nqual = sq[n].nqual; - } else { - pq[n].qualifiers = NULL; - pq[n].nqual = 0; } - } - break; - default: - return PMIX_ERR_UNKNOWN_DATA_TYPE; - } - break; - case PMIX_POINTER: - memcpy(&p->data.ptr, &src->data.ptr, sizeof(void*)); - break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - p->data.array->size = src->data.array->size; - if (0 < src->data.array->size) { - p->data.array->array = (pmix_info_t*)malloc(src->data.array->size * sizeof(pmix_info_t)); - if (NULL == p->data.array->array) { - return PMIX_ERR_NOMEM; + break; + case PMIX_INFO: + PMIX_INFO_CREATE(p->data.darray->array, src->data.darray->size); + p1 = (pmix_info_t*)p->data.darray->array; + s1 = (pmix_info_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type); + } + break; + case PMIX_PDATA: + PMIX_PDATA_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pd = (pmix_pdata_t*)p->data.darray->array; + sd = (pmix_pdata_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + PMIX_PDATA_LOAD(&pd[n], &sd[n].proc, sd[n].key, &sd[n].value.data.flag, sd[n].value.type); + } + break; + case PMIX_BUFFER: + p->data.darray->array = (pmix_buffer_t*)malloc(src->data.darray->size * sizeof(pmix_buffer_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pb = (pmix_buffer_t*)p->data.darray->array; + sb = (pmix_buffer_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + PMIX_CONSTRUCT(&pb[n], pmix_buffer_t); + pmix20_bfrop_copy_payload(&pb[n], &sb[n]); + } + break; + case PMIX_BYTE_OBJECT: + case PMIX_COMPRESSED_STRING: + p->data.darray->array = (pmix_byte_object_t*)malloc(src->data.darray->size * sizeof(pmix_byte_object_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pbo = (pmix_byte_object_t*)p->data.darray->array; + sbo = (pmix_byte_object_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != sbo[n].bytes && 0 < sbo[n].size) { + pbo[n].size = sbo[n].size; + pbo[n].bytes = (char*)malloc(pbo[n].size); + memcpy(pbo[n].bytes, sbo[n].bytes, pbo[n].size); + } else { + pbo[n].bytes = NULL; + pbo[n].size = 0; + } + } + break; + case PMIX_KVAL: + p->data.darray->array = (pmix_kval_t*)calloc(src->data.darray->size , sizeof(pmix_kval_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pk = (pmix_kval_t*)p->data.darray->array; + sk = (pmix_kval_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != sk[n].key) { + pk[n].key = strdup(sk[n].key); + } + if (NULL != sk[n].value) { + PMIX_VALUE_CREATE(pk[n].value, 1); + if (NULL == pk[n].value) { + free(p->data.darray->array); + return PMIX_ERR_NOMEM; + } + if (PMIX_SUCCESS != (rc = pmix20_bfrop_value_xfer(pk[n].value, sk[n].value))) { + return rc; + } + } + } + break; + case PMIX_MODEX: + PMIX_MODEX_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pm = (pmix_modex_data_t*)p->data.darray->array; + sm = (pmix_modex_data_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + memcpy(&pm[n], &sm[n], sizeof(pmix_modex_data_t)); + if (NULL != sm[n].blob && 0 < sm[n].size) { + pm[n].blob = (uint8_t*)malloc(sm[n].size); + if (NULL == pm[n].blob) { + return PMIX_ERR_NOMEM; + } + memcpy(pm[n].blob, sm[n].blob, sm[n].size); + pm[n].size = sm[n].size; + } else { + pm[n].blob = NULL; + pm[n].size = 0; + } + } + break; + case PMIX_PERSIST: + p->data.darray->array = (pmix_persistence_t*)malloc(src->data.darray->size * sizeof(pmix_persistence_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_persistence_t)); + break; + case PMIX_POINTER: + p->data.darray->array = (char**)malloc(src->data.darray->size * sizeof(char*)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + prarray = (char**)p->data.darray->array; + strarray = (char**)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + prarray[n] = strarray[n]; + } + break; + case PMIX_SCOPE: + p->data.darray->array = (pmix_scope_t*)malloc(src->data.darray->size * sizeof(pmix_scope_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_scope_t)); + break; + case PMIX_DATA_RANGE: + p->data.darray->array = (pmix_data_range_t*)malloc(src->data.darray->size * sizeof(pmix_data_range_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_data_range_t)); + break; + case PMIX_COMMAND: + p->data.darray->array = (pmix_cmd_t*)malloc(src->data.darray->size * sizeof(pmix_cmd_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_cmd_t)); + break; + case PMIX_INFO_DIRECTIVES: + p->data.darray->array = (pmix_info_directives_t*)malloc(src->data.darray->size * sizeof(pmix_info_directives_t)); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + memcpy(p->data.darray->array, src->data.darray->array, src->data.darray->size * sizeof(pmix_info_directives_t)); + break; + case PMIX_PROC_INFO: + PMIX_PROC_INFO_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pi = (pmix_proc_info_t*)p->data.darray->array; + si = (pmix_proc_info_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + memcpy(&pi[n].proc, &si[n].proc, sizeof(pmix_proc_t)); + if (NULL != si[n].hostname) { + pi[n].hostname = strdup(si[n].hostname); + } else { + pi[n].hostname = NULL; + } + if (NULL != si[n].executable_name) { + pi[n].executable_name = strdup(si[n].executable_name); + } else { + pi[n].executable_name = NULL; + } + pi[n].pid = si[n].pid; + pi[n].exit_code = si[n].exit_code; + pi[n].state = si[n].state; + } + break; + case PMIX_DATA_ARRAY: + return PMIX_ERR_NOT_SUPPORTED; // don't support iterative arrays + case PMIX_QUERY: + PMIX_QUERY_CREATE(p->data.darray->array, src->data.darray->size); + if (NULL == p->data.darray->array) { + return PMIX_ERR_NOMEM; + } + pq = (pmix_query_t*)p->data.darray->array; + sq = (pmix_query_t*)src->data.darray->array; + for (n=0; n < src->data.darray->size; n++) { + if (NULL != sq[n].keys) { + pq[n].keys = pmix_argv_copy(sq[n].keys); + } + if (NULL != sq[n].qualifiers && 0 < sq[n].nqual) { + PMIX_INFO_CREATE(pq[n].qualifiers, sq[n].nqual); + if (NULL == pq[n].qualifiers) { + PMIX_QUERY_FREE(pq, src->data.darray->size); + return PMIX_ERR_NOMEM; + } + for (m=0; m < sq[n].nqual; m++) { + PMIX_INFO_XFER(&pq[n].qualifiers[m], &sq[n].qualifiers[m]); + } + pq[n].nqual = sq[n].nqual; + } else { + pq[n].qualifiers = NULL; + pq[n].nqual = 0; + } + } + break; + default: + return PMIX_ERR_UNKNOWN_DATA_TYPE; } - p1 = (pmix_info_t*)p->data.array->array; - s1 = (pmix_info_t*)src->data.array->array; - for (n=0; n < src->data.darray->size; n++) { - PMIX_INFO_LOAD(&p1[n], s1[n].key, &s1[n].value.data.flag, s1[n].value.type); - } - } - break; - /********************/ - default: - pmix_output(0, "COPY-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)src->type); - return PMIX_ERROR; + break; + case PMIX_POINTER: + memcpy(&p->data.ptr, &src->data.ptr, sizeof(void*)); + break; + /**** DEPRECATED ****/ + case PMIX_INFO_ARRAY: + return PMIX_ERR_NOT_SUPPORTED; + /********************/ + default: + pmix_output(0, "COPY-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)src->type); + return PMIX_ERROR; } return PMIX_SUCCESS; } @@ -895,7 +883,7 @@ pmix_status_t pmix20_bfrop_copy_info(pmix_info_t **dest, pmix_info_t *src, pmix_data_type_t type) { *dest = (pmix_info_t*)malloc(sizeof(pmix_info_t)); - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); (*dest)->flags = src->flags; return pmix20_bfrop_value_xfer(&(*dest)->value, &src->value); } @@ -924,7 +912,7 @@ pmix_status_t pmix20_bfrop_copy_app(pmix_app_t **dest, pmix_app_t *src, (*dest)->ninfo = src->ninfo; (*dest)->info = (pmix_info_t*)malloc(src->ninfo * sizeof(pmix_info_t)); for (j=0; j < src->ninfo; j++) { - (void)strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->info[j].key, src->info[j].key, PMIX_MAX_KEYLEN); pmix20_bfrop_value_xfer(&(*dest)->info[j].value, &src->info[j].value); } return PMIX_SUCCESS; @@ -955,7 +943,7 @@ pmix_status_t pmix20_bfrop_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; } - (void)strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->nspace, src->nspace, PMIX_MAX_NSLEN); (*dest)->rank = src->rank; return PMIX_SUCCESS; } @@ -1008,9 +996,9 @@ pmix_status_t pmix20_bfrop_copy_pdata(pmix_pdata_t **dest, pmix_pdata_t *src, pmix_data_type_t type) { *dest = (pmix_pdata_t*)malloc(sizeof(pmix_pdata_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; - (void)strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); + pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); return pmix20_bfrop_value_xfer(&(*dest)->value, &src->value); } @@ -1018,7 +1006,7 @@ pmix_status_t pmix20_bfrop_copy_pinfo(pmix_proc_info_t **dest, pmix_proc_info_t pmix_data_type_t type) { *dest = (pmix_proc_info_t*)malloc(sizeof(pmix_proc_info_t)); - (void)strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); + pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; if (NULL != src->hostname) { (*dest)->hostname = strdup(src->hostname); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/internal.h b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/internal.h index d85ac2985c..1478687f6c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/internal.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/internal.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -42,6 +42,55 @@ BEGIN_C_DECLS +/* DEPRECATED data type values */ +#define PMIX_MODEX 29 +#define PMIX_INFO_ARRAY 44 + +/**** PMIX MODEX STRUCT - DEPRECATED ****/ +typedef struct pmix_modex_data { + char nspace[PMIX_MAX_NSLEN+1]; + int rank; + uint8_t *blob; + size_t size; +} pmix_modex_data_t; +/* utility macros for working with pmix_modex_t structs */ +#define PMIX_MODEX_CREATE(m, n) \ + do { \ + (m) = (pmix_modex_data_t*)calloc((n) , sizeof(pmix_modex_data_t)); \ + } while (0) + +#define PMIX_MODEX_RELEASE(m) \ + do { \ + PMIX_MODEX_DESTRUCT((m)); \ + free((m)); \ + (m) = NULL; \ + } while (0) + +#define PMIX_MODEX_CONSTRUCT(m) \ + do { \ + memset((m), 0, sizeof(pmix_modex_data_t)); \ + } while (0) + +#define PMIX_MODEX_DESTRUCT(m) \ + do { \ + if (NULL != (m)->blob) { \ + free((m)->blob); \ + (m)->blob = NULL; \ + } \ + } while (0) + +#define PMIX_MODEX_FREE(m, n) \ + do { \ + size_t _s; \ + if (NULL != (m)) { \ + for (_s=0; _s < (n); _s++) { \ + PMIX_MODEX_DESTRUCT(&((m)[_s])); \ + } \ + free((m)); \ + (m) = NULL; \ + } \ + } while (0) + /* * Implementations of API functions */ @@ -59,7 +108,7 @@ pmix_status_t pmix20_bfrop_print(char **output, char *prefix, void *src, pmix_da pmix_status_t pmix20_bfrop_copy_payload(pmix_buffer_t *dest, pmix_buffer_t *src); -pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, pmix_value_t *src); +pmix_status_t pmix20_bfrop_value_xfer(pmix_value_t *p, const pmix_value_t *src); void pmix20_bfrop_value_load(pmix_value_t *v, const void *data, pmix_data_type_t type); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/pack.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/pack.c index 9163620449..6aa194b8b9 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/pack.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/pack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011-2013 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -613,13 +613,6 @@ static pmix_status_t pack_val(pmix_buffer_t *buffer, return ret; } break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - if (PMIX_SUCCESS != (ret = pmix20_bfrop_pack_buffer(buffer, p->data.array, 1, PMIX_INFO_ARRAY))) { - return ret; - } - break; - /********************/ default: pmix_output(0, "PACK-PMIX-VALUE: UNSUPPORTED TYPE %d", (int)p->type); return PMIX_ERROR; @@ -1035,10 +1028,9 @@ pmix_status_t pmix20_bfrop_pack_alloc_directive(pmix_buffer_t *buffer, const voi return pmix20_bfrop_pack_byte(buffer, src, num_vals, PMIX_UINT8); } - /**** DEPRECATED ****/ pmix_status_t pmix20_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, - int32_t num_vals, pmix_data_type_t type) + int32_t num_vals, pmix_data_type_t type) { pmix_info_array_t *ptr; int32_t i; @@ -1048,12 +1040,12 @@ pmix_status_t pmix20_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, for (i = 0; i < num_vals; ++i) { /* pack the size */ - if (PMIX_SUCCESS != (ret = pmix20_bfrop_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { return ret; } if (0 < ptr[i].size) { /* pack the values */ - if (PMIX_SUCCESS != (ret = pmix20_bfrop_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { return ret; } } @@ -1062,3 +1054,4 @@ pmix_status_t pmix20_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, return PMIX_SUCCESS; } /********************/ + diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/print.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/print.c index 500bd87f1c..a9009be364 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/print.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/print.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. @@ -761,7 +761,7 @@ pmix_status_t pmix20_bfrop_print_status(char **output, char *prefix, * PMIX_VALUE */ pmix_status_t pmix20_bfrop_print_value(char **output, char *prefix, - pmix_value_t *src, pmix_data_type_t type) + pmix_value_t *src, pmix_data_type_t type) { char *prefx; int rc; @@ -904,12 +904,6 @@ pmix_status_t pmix20_bfrop_print_status(char **output, char *prefix, rc = asprintf(output, "%sPMIX_VALUE: Data type: DATA_ARRAY\tARRAY SIZE: %ld", prefx, (long)src->data.darray->size); break; - /**** DEPRECATED ****/ - case PMIX_INFO_ARRAY: - rc = asprintf(output, "%sPMIX_VALUE: Data type: INFO_ARRAY\tARRAY SIZE: %ld", - prefx, (long)src->data.array->size); - break; - /********************/ default: rc = asprintf(output, "%sPMIX_VALUE: Data type: UNKNOWN\tValue: UNPRINTABLE", prefx); break; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/unpack.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/unpack.c index f812a005bc..91ce264fd3 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/unpack.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v20/unpack.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -756,12 +756,13 @@ pmix_status_t pmix20_bfrop_unpack_status(pmix_buffer_t *buffer, void *dest, break; /**** DEPRECATED ****/ case PMIX_INFO_ARRAY: - /* this field is now a pointer, so we must allocate storage for it */ - val->data.array = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); - if (NULL == val->data.array) { - return PMIX_ERR_NOMEM; - } - if (PMIX_SUCCESS != (ret = pmix20_bfrop_unpack_buffer(buffer, val->data.array, &m, PMIX_INFO_ARRAY))) { + /* we don't know anything about info array's so we + * have to convert this to a data array */ + val->data.darray = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); + val->data.darray->type = PMIX_INFO_ARRAY; + val->data.darray->size = m; + /* unpack into it */ + if (PMIX_SUCCESS != (ret = pmix20_bfrop_unpack_buffer(buffer, &val->data.darray->array, &m, PMIX_INFO_ARRAY))) { return ret; } break; @@ -825,7 +826,7 @@ pmix_status_t pmix20_bfrop_unpack_info(pmix_buffer_t *buffer, void *dest, PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack the flags */ m=1; @@ -882,7 +883,7 @@ pmix_status_t pmix20_bfrop_unpack_pdata(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); + pmix_strncpy(ptr[i].key, tmp, PMIX_MAX_KEYLEN); free(tmp); /* unpack value - since the value structure is statically-defined * instead of a pointer in this struct, we directly unpack it to @@ -962,7 +963,7 @@ pmix_status_t pmix20_bfrop_unpack_proc(pmix_buffer_t *buffer, void *dest, if (NULL == tmp) { return PMIX_ERROR; } - (void)strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); + pmix_strncpy(ptr[i].nspace, tmp, PMIX_MAX_NSLEN); free(tmp); /* unpack the rank */ m=1; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v21/bfrop_pmix21.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v21/bfrop_pmix21.c index adcc3cba56..8100b70ed6 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v21/bfrop_pmix21.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v21/bfrop_pmix21.c @@ -13,7 +13,7 @@ * Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -63,6 +63,37 @@ pmix_bfrops_module_t pmix_bfrops_pmix21_module = { .data_type_string = data_type_string }; +/* DEPRECATED data type values */ +#define PMIX_MODEX 29 +#define PMIX_INFO_ARRAY 44 + +/**** PMIX MODEX STRUCT - DEPRECATED ****/ +typedef struct pmix_modex_data { + char nspace[PMIX_MAX_NSLEN+1]; + int rank; + uint8_t *blob; + size_t size; +} pmix_modex_data_t; + +static pmix_status_t pmix21_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_pack_modex(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_unpack_array(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_copy_array(pmix_info_array_t **dest, + pmix_info_array_t *src, + pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_copy_modex(pmix_modex_data_t **dest, pmix_modex_data_t *src, + pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_print_array(char **output, char *prefix, + pmix_info_array_t *src, pmix_data_type_t type); +static pmix_status_t pmix21_bfrop_print_modex(char **output, char *prefix, + pmix_modex_data_t *src, pmix_data_type_t type); + + static pmix_status_t init(void) { /* some standard types don't require anything special */ @@ -265,10 +296,10 @@ static pmix_status_t init(void) &mca_bfrops_v21_component.types); PMIX_REGISTER_TYPE("PMIX_MODEX", PMIX_MODEX, - pmix_bfrops_base_pack_modex, - pmix_bfrops_base_unpack_modex, - pmix_bfrops_base_copy_modex, - pmix_bfrops_base_print_modex, + pmix21_bfrop_pack_modex, + pmix21_bfrop_unpack_modex, + pmix21_bfrop_copy_modex, + pmix21_bfrop_print_modex, &mca_bfrops_v21_component.types); /* these are fixed-sized values and can be done by base */ @@ -374,10 +405,10 @@ static pmix_status_t init(void) /**** DEPRECATED ****/ PMIX_REGISTER_TYPE("PMIX_INFO_ARRAY", PMIX_INFO_ARRAY, - pmix_bfrops_base_pack_array, - pmix_bfrops_base_unpack_array, - pmix_bfrops_base_copy_array, - pmix_bfrops_base_print_array, + pmix21_bfrop_pack_array, + pmix21_bfrop_unpack_array, + pmix21_bfrop_copy_array, + pmix21_bfrop_print_array, &mca_bfrops_v21_component.types); /********************/ @@ -446,3 +477,198 @@ static const char* data_type_string(pmix_data_type_t type) { return pmix_bfrops_base_data_type_string(&mca_bfrops_v21_component.types, type); } + +/**** DEPRECATED ****/ +static pmix_status_t pmix21_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type) +{ + pmix_info_array_t *ptr; + int32_t i; + pmix_status_t ret; + + ptr = (pmix_info_array_t *) src; + + for (i = 0; i < num_vals; ++i) { + /* pack the size */ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + /* pack the values */ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { + return ret; + } + } + } + + return PMIX_SUCCESS; +} + +static pmix_status_t pmix21_bfrop_pack_modex(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type) +{ + pmix_modex_data_t *ptr; + int32_t i; + pmix_status_t ret; + + ptr = (pmix_modex_data_t *) src; + + for (i = 0; i < num_vals; ++i) { + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { + return ret; + } + if( 0 < ptr[i].size){ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_byte(buffer, ptr[i].blob, ptr[i].size, PMIX_UINT8))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + +/********************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix21_bfrop_unpack_array(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) +{ + pmix_info_array_t *ptr; + int32_t i, n, m; + pmix_status_t ret; + + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix21_bfrop_unpack: %d info arrays", *num_vals); + + ptr = (pmix_info_array_t*) dest; + n = *num_vals; + + for (i = 0; i < n; ++i) { + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix21_bfrop_unpack: init array[%d]", i); + memset(&ptr[i], 0, sizeof(pmix_info_array_t)); + /* unpack the size of this array */ + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + ptr[i].array = (pmix_info_t*)malloc(ptr[i].size * sizeof(pmix_info_t)); + m=ptr[i].size; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_value(buffer, ptr[i].array, &m, PMIX_INFO))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + +static pmix_status_t pmix21_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) +{ + pmix_modex_data_t *ptr; + int32_t i, n, m; + pmix_status_t ret; + + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix20_bfrop_unpack: %d modex", *num_vals); + + ptr = (pmix_modex_data_t *) dest; + n = *num_vals; + + for (i = 0; i < n; ++i) { + memset(&ptr[i], 0, sizeof(pmix_modex_data_t)); + /* unpack the number of bytes */ + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + ptr[i].blob = (uint8_t*)malloc(ptr[i].size * sizeof(uint8_t)); + m=ptr[i].size; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_byte(buffer, ptr[i].blob, &m, PMIX_UINT8))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + +/********************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix21_bfrop_copy_array(pmix_info_array_t **dest, + pmix_info_array_t *src, + pmix_data_type_t type) +{ + pmix_info_t *d1, *s1; + + *dest = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); + (*dest)->size = src->size; + (*dest)->array = (pmix_info_t*)malloc(src->size * sizeof(pmix_info_t)); + d1 = (pmix_info_t*)(*dest)->array; + s1 = (pmix_info_t*)src->array; + memcpy(d1, s1, src->size * sizeof(pmix_info_t)); + return PMIX_SUCCESS; +} + +static pmix_status_t pmix21_bfrop_copy_modex(pmix_modex_data_t **dest, pmix_modex_data_t *src, + pmix_data_type_t type) +{ + *dest = (pmix_modex_data_t*)malloc(sizeof(pmix_modex_data_t)); + if (NULL == *dest) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + (*dest)->blob = NULL; + (*dest)->size = 0; + if (NULL != src->blob) { + (*dest)->blob = (uint8_t*)malloc(src->size * sizeof(uint8_t)); + if (NULL == (*dest)->blob) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + memcpy((*dest)->blob, src->blob, src->size * sizeof(uint8_t)); + (*dest)->size = src->size; + } + return PMIX_SUCCESS; +} + +/*******************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix21_bfrop_print_array(char **output, char *prefix, + pmix_info_array_t *src, pmix_data_type_t type) +{ + size_t j; + char *tmp, *tmp2, *tmp3, *pfx; + pmix_info_t *s1; + + if (0 > asprintf(&tmp, "%sARRAY SIZE: %ld", prefix, (long)src->size)) { + return PMIX_ERR_NOMEM; + } + if (0 > asprintf(&pfx, "\n%s\t", (NULL == prefix) ? "" : prefix)) { + free(tmp); + return PMIX_ERR_NOMEM; + } + s1 = (pmix_info_t*)src->array; + + for (j=0; j < src->size; j++) { + pmix_bfrops_base_print_info(&tmp2, pfx, &s1[j], PMIX_INFO); + if (0 > asprintf(&tmp3, "%s%s", tmp, tmp2)) { + free(tmp); + free(tmp2); + return PMIX_ERR_NOMEM; + } + free(tmp); + free(tmp2); + tmp = tmp3; + } + *output = tmp; + return PMIX_SUCCESS; +} + +static pmix_status_t pmix21_bfrop_print_modex(char **output, char *prefix, + pmix_modex_data_t *src, pmix_data_type_t type) +{ + return PMIX_SUCCESS; +} + +/********************/ diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v3/bfrop_pmix3.c b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v3/bfrop_pmix3.c index 828cb39ac6..eac138a80d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v3/bfrop_pmix3.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/bfrops/v3/bfrop_pmix3.c @@ -63,6 +63,37 @@ pmix_bfrops_module_t pmix_bfrops_pmix3_module = { .data_type_string = data_type_string }; +/* DEPRECATED data type values */ +#define PMIX_MODEX 29 +#define PMIX_INFO_ARRAY 44 + +/**** PMIX MODEX STRUCT - DEPRECATED ****/ +typedef struct pmix_modex_data { + char nspace[PMIX_MAX_NSLEN+1]; + int rank; + uint8_t *blob; + size_t size; +} pmix_modex_data_t; + +static pmix_status_t pmix3_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +static pmix_status_t pmix3_bfrop_pack_modex(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type); +static pmix_status_t pmix3_bfrop_unpack_array(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); +static pmix_status_t pmix3_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type); +static pmix_status_t pmix3_bfrop_copy_array(pmix_info_array_t **dest, + pmix_info_array_t *src, + pmix_data_type_t type); +static pmix_status_t pmix3_bfrop_copy_modex(pmix_modex_data_t **dest, pmix_modex_data_t *src, + pmix_data_type_t type); +static pmix_status_t pmix3_bfrop_print_array(char **output, char *prefix, + pmix_info_array_t *src, pmix_data_type_t type); +static pmix_status_t pmix3_bfrop_print_modex(char **output, char *prefix, + pmix_modex_data_t *src, pmix_data_type_t type); + + static pmix_status_t init(void) { /* some standard types don't require anything special */ @@ -265,10 +296,10 @@ static pmix_status_t init(void) &mca_bfrops_v3_component.types); PMIX_REGISTER_TYPE("PMIX_MODEX", PMIX_MODEX, - pmix_bfrops_base_pack_modex, - pmix_bfrops_base_unpack_modex, - pmix_bfrops_base_copy_modex, - pmix_bfrops_base_print_modex, + pmix3_bfrop_pack_modex, + pmix3_bfrop_unpack_modex, + pmix3_bfrop_copy_modex, + pmix3_bfrop_print_modex, &mca_bfrops_v3_component.types); /* these are fixed-sized values and can be done by base */ @@ -390,10 +421,10 @@ static pmix_status_t init(void) /**** DEPRECATED ****/ PMIX_REGISTER_TYPE("PMIX_INFO_ARRAY", PMIX_INFO_ARRAY, - pmix_bfrops_base_pack_array, - pmix_bfrops_base_unpack_array, - pmix_bfrops_base_copy_array, - pmix_bfrops_base_print_array, + pmix3_bfrop_pack_array, + pmix3_bfrop_unpack_array, + pmix3_bfrop_copy_array, + pmix3_bfrop_print_array, &mca_bfrops_v3_component.types); /********************/ @@ -462,3 +493,201 @@ static const char* data_type_string(pmix_data_type_t type) { return pmix_bfrops_base_data_type_string(&mca_bfrops_v3_component.types, type); } + +/**** DEPRECATED ****/ +static pmix_status_t pmix3_bfrop_pack_array(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type) +{ + pmix_info_array_t *ptr; + int32_t i; + pmix_status_t ret; + + ptr = (pmix_info_array_t *) src; + + for (i = 0; i < num_vals; ++i) { + /* pack the size */ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + /* pack the values */ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_info(buffer, ptr[i].array, ptr[i].size, PMIX_INFO))) { + return ret; + } + } + } + + return PMIX_SUCCESS; +} + +static pmix_status_t pmix3_bfrop_pack_modex(pmix_buffer_t *buffer, const void *src, + int32_t num_vals, pmix_data_type_t type) +{ + pmix_modex_data_t *ptr; + int32_t i; + pmix_status_t ret; + + ptr = (pmix_modex_data_t *) src; + + for (i = 0; i < num_vals; ++i) { + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_sizet(buffer, &ptr[i].size, 1, PMIX_SIZE))) { + return ret; + } + if( 0 < ptr[i].size){ + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_pack_byte(buffer, ptr[i].blob, ptr[i].size, PMIX_UINT8))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + + +/********************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix3_bfrop_unpack_array(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) +{ + pmix_info_array_t *ptr; + int32_t i, n, m; + pmix_status_t ret; + + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix3_bfrop_unpack: %d info arrays", *num_vals); + + ptr = (pmix_info_array_t*) dest; + n = *num_vals; + + for (i = 0; i < n; ++i) { + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix3_bfrop_unpack: init array[%d]", i); + memset(&ptr[i], 0, sizeof(pmix_info_array_t)); + /* unpack the size of this array */ + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + ptr[i].array = (pmix_info_t*)malloc(ptr[i].size * sizeof(pmix_info_t)); + m=ptr[i].size; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_value(buffer, ptr[i].array, &m, PMIX_INFO))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + +static pmix_status_t pmix3_bfrop_unpack_modex(pmix_buffer_t *buffer, void *dest, + int32_t *num_vals, pmix_data_type_t type) +{ + pmix_modex_data_t *ptr; + int32_t i, n, m; + pmix_status_t ret; + + pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, + "pmix20_bfrop_unpack: %d modex", *num_vals); + + ptr = (pmix_modex_data_t *) dest; + n = *num_vals; + + for (i = 0; i < n; ++i) { + memset(&ptr[i], 0, sizeof(pmix_modex_data_t)); + /* unpack the number of bytes */ + m=1; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_sizet(buffer, &ptr[i].size, &m, PMIX_SIZE))) { + return ret; + } + if (0 < ptr[i].size) { + ptr[i].blob = (uint8_t*)malloc(ptr[i].size * sizeof(uint8_t)); + m=ptr[i].size; + if (PMIX_SUCCESS != (ret = pmix_bfrops_base_unpack_byte(buffer, ptr[i].blob, &m, PMIX_UINT8))) { + return ret; + } + } + } + return PMIX_SUCCESS; +} + + +/********************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix3_bfrop_copy_array(pmix_info_array_t **dest, + pmix_info_array_t *src, + pmix_data_type_t type) +{ + pmix_info_t *d1, *s1; + + *dest = (pmix_info_array_t*)malloc(sizeof(pmix_info_array_t)); + (*dest)->size = src->size; + (*dest)->array = (pmix_info_t*)malloc(src->size * sizeof(pmix_info_t)); + d1 = (pmix_info_t*)(*dest)->array; + s1 = (pmix_info_t*)src->array; + memcpy(d1, s1, src->size * sizeof(pmix_info_t)); + return PMIX_SUCCESS; +} + +static pmix_status_t pmix3_bfrop_copy_modex(pmix_modex_data_t **dest, pmix_modex_data_t *src, + pmix_data_type_t type) +{ + *dest = (pmix_modex_data_t*)malloc(sizeof(pmix_modex_data_t)); + if (NULL == *dest) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + (*dest)->blob = NULL; + (*dest)->size = 0; + if (NULL != src->blob) { + (*dest)->blob = (uint8_t*)malloc(src->size * sizeof(uint8_t)); + if (NULL == (*dest)->blob) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + memcpy((*dest)->blob, src->blob, src->size * sizeof(uint8_t)); + (*dest)->size = src->size; + } + return PMIX_SUCCESS; +} + +/*******************/ + +/**** DEPRECATED ****/ +static pmix_status_t pmix3_bfrop_print_array(char **output, char *prefix, + pmix_info_array_t *src, pmix_data_type_t type) +{ + size_t j; + char *tmp, *tmp2, *tmp3, *pfx; + pmix_info_t *s1; + + if (0 > asprintf(&tmp, "%sARRAY SIZE: %ld", prefix, (long)src->size)) { + return PMIX_ERR_NOMEM; + } + if (0 > asprintf(&pfx, "\n%s\t", (NULL == prefix) ? "" : prefix)) { + free(tmp); + return PMIX_ERR_NOMEM; + } + s1 = (pmix_info_t*)src->array; + + for (j=0; j < src->size; j++) { + pmix_bfrops_base_print_info(&tmp2, pfx, &s1[j], PMIX_INFO); + if (0 > asprintf(&tmp3, "%s%s", tmp, tmp2)) { + free(tmp); + free(tmp2); + return PMIX_ERR_NOMEM; + } + free(tmp); + free(tmp2); + tmp = tmp3; + } + *output = tmp; + return PMIX_SUCCESS; +} + +static pmix_status_t pmix3_bfrop_print_modex(char **output, char *prefix, + pmix_modex_data_t *src, pmix_data_type_t type) +{ + return PMIX_SUCCESS; +} + + +/********************/ diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/common/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/common/Makefile.am new file mode 100644 index 0000000000..4567c65430 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/Makefile.am @@ -0,0 +1,25 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Note that this file must exist, even though it is empty (there is no +# "base" directory for the common framework). autogen.pl and +# opal_mca.m4 assume that every framework has a top-level Makefile.am. +# We *could* adjust the framework glue code to exclude "common" from +# this requirement, but it's just a lot easier to have an empty +# Makefile.am here. diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/Makefile.am new file mode 100644 index 0000000000..50c601838f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/Makefile.am @@ -0,0 +1,59 @@ +# +# Copyright (c) 2018 Mellanox Technologies. All rights reserved. +# +# Copyright (c) 2018 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# Header files + +AM_CPPFLAGS = $(LTDLINCL) + +headers = \ + dstore_common.h \ + dstore_base.h \ + dstore_segment.h \ + dstore_file.h + +# Source files + +sources = \ + dstore_base.c \ + dstore_segment.c + +lib_LTLIBRARIES = +noinst_LTLIBRARIES = +comp_inst = libmca_common_dstore.la +comp_noinst = libmca_common_dstore_noinst.la + +if MCA_BUILD_pmix_common_dstore_DSO +lib_LTLIBRARIES += $(comp_inst) +else +noinst_LTLIBRARIES += $(comp_noinst) +endif + +libmca_common_dstore_la_SOURCES = $(headers) $(sources) +libmca_common_dstore_la_LDFLAGS = -version-info $(libmca_common_dstore_so_version) +libmca_common_dstore_noinst_la_SOURCES = $(headers) $(sources) + +# Conditionally install the header files + +if WANT_INSTALL_HEADERS +pmixdir = $(pmixincludedir)/$(subdir) +pmix_HEADERS = $(headers) +endif + +all-local: + if test -z "$(lib_LTLIBRARIES)"; then \ + rm -f "$(comp_inst)"; \ + $(LN_S) "$(comp_noinst)" "$(comp_inst)"; \ + fi + +clean-local: + if test -z "$(lib_LTLIBRARIES)"; then \ + rm -f "$(comp_inst)"; \ + fi diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/configure.m4 b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/configure.m4 new file mode 100644 index 0000000000..ca90b6568f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/configure.m4 @@ -0,0 +1,17 @@ +# -*- shell-script -*- +# +# Copyright (c) 2018 Mellanox Technologies. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# MCA_pmix_common_dstore_CONFIG([action-if-can-compile], +# [action-if-cant-compile]) +# ------------------------------------------------ +AC_DEFUN([MCA_pmix_common_dstore_CONFIG], [ + AC_CONFIG_FILES([src/mca/common/dstore/Makefile]) + $1 +])dnl diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_base.c similarity index 55% rename from opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c rename to opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_base.c index 5246541225..c0fc676e6c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_base.c @@ -1,9 +1,9 @@ /* * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. - * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2018 Research Organization for Information Science + * Copyright (c) 2018-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ @@ -50,8 +50,10 @@ #include "src/mca/preg/preg.h" #include "src/mca/gds/base/base.h" -#include "gds_dstore.h" #include "src/mca/pshmem/base/base.h" +#include "dstore_common.h" +#include "dstore_base.h" +#include "dstore_segment.h" #define ESH_REGION_EXTENSION "EXTENSION_SLOT" #define ESH_REGION_INVALIDATED "INVALIDATED" @@ -60,418 +62,89 @@ #define ESH_ENV_NS_DATA_SEG_SIZE "NS_DATA_SEG_SIZE" #define ESH_ENV_LINEAR "SM_USE_LINEAR_SEARCH" -#define ESH_MIN_KEY_LEN (sizeof(ESH_REGION_INVALIDATED)) - -#define ESH_KV_SIZE(addr) \ -__pmix_attribute_extension__ ({ \ - size_t sz; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - sz = ESH_KV_SIZE_V12(addr); \ - } else { \ - sz = ESH_KV_SIZE_V20(addr); \ - } \ - sz; \ -}) - -#define ESH_KNAME_PTR(addr) \ -__pmix_attribute_extension__ ({ \ - char *name_ptr; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - name_ptr = ESH_KNAME_PTR_V12(addr); \ - } else { \ - name_ptr = ESH_KNAME_PTR_V20(addr); \ - } \ - name_ptr; \ -}) - -#define ESH_KNAME_LEN(key) \ -__pmix_attribute_extension__ ({ \ - size_t len; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - len = ESH_KNAME_LEN_V12(key); \ - } else { \ - len = ESH_KNAME_LEN_V20(key); \ - } \ - len; \ -}) - -#define ESH_DATA_PTR(addr) \ -__pmix_attribute_extension__ ({ \ - uint8_t *data_ptr; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - data_ptr = ESH_DATA_PTR_V12(addr); \ - } else { \ - data_ptr = ESH_DATA_PTR_V20(addr); \ - } \ - data_ptr; \ -}) - -#define ESH_DATA_SIZE(addr, data_ptr) \ -__pmix_attribute_extension__ ({ \ - size_t sz; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - sz = ESH_DATA_SIZE_V12(addr); \ - } else { \ - sz = ESH_DATA_SIZE_V20(addr, data_ptr); \ - } \ - sz; \ -}) - -#define ESH_KEY_SIZE(key, size) \ -__pmix_attribute_extension__ ({ \ - size_t len; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - len = ESH_KEY_SIZE_V12(key, size); \ - } else { \ - len = ESH_KEY_SIZE_V20(key, size); \ - } \ - len; \ -}) - -#define EXT_SLOT_SIZE() \ -__pmix_attribute_extension__ ({ \ - size_t sz; \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - sz = EXT_SLOT_SIZE_V12(); \ - } else { \ - sz = EXT_SLOT_SIZE_V20(); \ - } \ - sz; \ -}) - -#define ESH_PUT_KEY(addr, key, buffer, size) \ -__pmix_attribute_extension__ ({ \ - if (PMIX_PROC_IS_V1(_client_peer())) { \ - ESH_PUT_KEY_V12(addr, key, buffer, size); \ - } else { \ - ESH_PUT_KEY_V20(addr, key, buffer, size); \ - } \ -}) - -/* PMIx v2.x dstore specific macro */ -#define ESH_KV_SIZE_V20(addr) \ -__pmix_attribute_extension__ ({ \ - size_t sz; \ - memcpy(&sz, addr, sizeof(size_t)); \ - sz; \ -}) - -#define ESH_KNAME_PTR_V20(addr) \ -__pmix_attribute_extension__ ({ \ - char *name_ptr = (char *)addr + sizeof(size_t); \ - name_ptr; \ -}) - -#define ESH_KNAME_LEN_V20(key) \ -__pmix_attribute_extension__ ({ \ - size_t kname_len = strlen(key) + 1; \ - size_t len = (kname_len < ESH_MIN_KEY_LEN) ? \ - ESH_MIN_KEY_LEN : kname_len; \ - len; \ -}) - -#define ESH_DATA_PTR_V20(addr) \ -__pmix_attribute_extension__ ({ \ - size_t kname_len = \ - ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr)); \ - uint8_t *data_ptr = addr + sizeof(size_t) + kname_len; \ - data_ptr; \ -}) - -#define ESH_DATA_SIZE_V20(addr, data_ptr) \ -__pmix_attribute_extension__ ({ \ - size_t sz = ESH_KV_SIZE_V20(addr); \ - size_t data_size = sz - (data_ptr - addr); \ - data_size; \ -}) - -#define ESH_KEY_SIZE_V20(key, size) \ -__pmix_attribute_extension__ ({ \ - size_t len = \ - sizeof(size_t) + ESH_KNAME_LEN_V20(key) + size; \ - len; \ -}) - -/* in ext slot new offset will be stored in case if - * new data were added for the same process during - * next commit - */ -#define EXT_SLOT_SIZE_V20() \ - (ESH_KEY_SIZE_V20(ESH_REGION_EXTENSION, sizeof(size_t))) - - -#define ESH_PUT_KEY_V20(addr, key, buffer, size) \ -__pmix_attribute_extension__ ({ \ - size_t sz = ESH_KEY_SIZE_V20(key, size); \ - memcpy(addr, &sz, sizeof(size_t)); \ - memset(addr + sizeof(size_t), 0, \ - ESH_KNAME_LEN_V20(key)); \ - strncpy((char *)addr + sizeof(size_t), \ - key, ESH_KNAME_LEN_V20(key)); \ - memcpy(addr + sizeof(size_t) + ESH_KNAME_LEN_V20(key), \ - buffer, size); \ -}) - -/* PMIx v1.2 dstore specific macro */ -#define ESH_KEY_SIZE_V12(key, size) \ -__pmix_attribute_extension__ ({ \ - size_t len = strlen(key) + 1 + sizeof(size_t) + size; \ - len; \ -}) - -/* in ext slot new offset will be stored in case if - * new data were added for the same process during - * next commit - */ -#define EXT_SLOT_SIZE_V12() \ - (ESH_KEY_SIZE_V12(ESH_REGION_EXTENSION, sizeof(size_t))) - -#define ESH_KV_SIZE_V12(addr) \ -__pmix_attribute_extension__ ({ \ - size_t sz; \ - memcpy(&sz, addr + \ - ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), \ - sizeof(size_t)); \ - sz += ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)) + \ - sizeof(size_t); \ - sz; \ -}) - -#define ESH_KNAME_PTR_V12(addr) \ -__pmix_attribute_extension__ ({ \ - char *name_ptr = (char *)addr; \ - name_ptr; \ -}) - -#define ESH_KNAME_LEN_V12(key) \ -__pmix_attribute_extension__ ({ \ - size_t len = strlen((char*)key) + 1; \ - len; \ -}) - -#define ESH_DATA_PTR_V12(addr) \ -__pmix_attribute_extension__ ({ \ - uint8_t *data_ptr = \ - addr + \ - sizeof(size_t) + \ - ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)); \ - data_ptr; \ -}) - -#define ESH_DATA_SIZE_V12(addr) \ -__pmix_attribute_extension__ ({ \ - size_t data_size; \ - memcpy(&data_size, \ - addr + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), \ - sizeof(size_t)); \ - data_size; \ -}) - -#define ESH_PUT_KEY_V12(addr, key, buffer, size) \ -__pmix_attribute_extension__ ({ \ - size_t sz = size; \ - memset(addr, 0, ESH_KNAME_LEN_V12(key)); \ - strncpy((char *)addr, key, ESH_KNAME_LEN_V12(key)); \ - memcpy(addr + ESH_KNAME_LEN_V12(key), &sz, \ - sizeof(size_t)); \ - memcpy(addr + ESH_KNAME_LEN_V12(key) + sizeof(size_t), \ - buffer, size); \ -}) - -#ifdef ESH_PTHREAD_LOCK -#define _ESH_LOCK(rwlock, func) \ -__pmix_attribute_extension__ ({ \ - pmix_status_t ret = PMIX_SUCCESS; \ - int rc; \ - rc = pthread_rwlock_##func(rwlock); \ - if (0 != rc) { \ - switch (errno) { \ - case EINVAL: \ - ret = PMIX_ERR_INIT; \ - break; \ - case EPERM: \ - ret = PMIX_ERR_NO_PERMISSIONS; \ - break; \ - } \ - } \ - if (ret) { \ - pmix_output(0, "%s %d:%s lock failed: %s", \ - __FILE__, __LINE__, __func__, strerror(errno)); \ - } \ - ret; \ -}) - -#define _ESH_WRLOCK(rwlock) _ESH_LOCK(rwlock, wrlock) -#define _ESH_RDLOCK(rwlock) _ESH_LOCK(rwlock, rdlock) -#define _ESH_UNLOCK(rwlock) _ESH_LOCK(rwlock, unlock) -#endif - -#ifdef ESH_FCNTL_LOCK -#define _ESH_LOCK(lockfd, operation) \ -__pmix_attribute_extension__ ({ \ - pmix_status_t ret = PMIX_SUCCESS; \ - int i; \ - struct flock fl = {0}; \ - fl.l_type = operation; \ - fl.l_whence = SEEK_SET; \ - for(i = 0; i < 10; i++) { \ - if( 0 > fcntl(lockfd, F_SETLKW, &fl) ) { \ - switch( errno ){ \ - case EINTR: \ - continue; \ - case ENOENT: \ - case EINVAL: \ - ret = PMIX_ERR_NOT_FOUND; \ - break; \ - case EBADF: \ - ret = PMIX_ERR_BAD_PARAM; \ - break; \ - case EDEADLK: \ - case EFAULT: \ - case ENOLCK: \ - ret = PMIX_ERR_RESOURCE_BUSY; \ - break; \ - default: \ - ret = PMIX_ERROR; \ - break; \ - } \ - } \ - break; \ - } \ - if (ret) { \ - pmix_output(0, "%s %d:%s lock failed: %s", \ - __FILE__, __LINE__, __func__, strerror(errno)); \ - } \ - ret; \ -}) - -#define _ESH_WRLOCK(lock) _ESH_LOCK(lock, F_WRLCK) -#define _ESH_RDLOCK(lock) _ESH_LOCK(lock, F_RDLCK) -#define _ESH_UNLOCK(lock) _ESH_LOCK(lock, F_UNLCK) -#endif - #define ESH_INIT_SESSION_TBL_SIZE 2 #define ESH_INIT_NS_MAP_TBL_SIZE 2 -static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_buffer_t *buf); -static seg_desc_t *_create_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id); -static seg_desc_t *_attach_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id); -static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info); -static int _put_ns_info_to_initial_segment(const ns_map_data_t *ns_map, pmix_pshmem_seg_t *metaseg, pmix_pshmem_seg_t *dataseg); -static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_map); -static ns_track_elem_t *_get_track_elem_for_namespace(ns_map_data_t *ns_map); -static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc); -static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset); -static void _update_initial_segment_info(const ns_map_data_t *ns_map); -static void _set_constants_from_env(void); -static void _delete_sm_desc(seg_desc_t *desc); -static int _pmix_getpagesize(void); -static inline ssize_t _get_univ_size(const char *nspace); +static int _store_data_for_rank(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, + pmix_rank_t rank, pmix_buffer_t *buf); +static int _update_ns_elem(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_elem, ns_seg_info_t *info); +static int _put_ns_info_to_initial_segment(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map, pmix_pshmem_seg_t *metaseg, + pmix_pshmem_seg_t *dataseg); +static ns_seg_info_t *_get_ns_info_from_initial_segment(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map); +static ns_track_elem_t *_get_track_elem_for_namespace(pmix_common_dstore_ctx_t *ds_ctx, + ns_map_data_t *ns_map); +static rank_meta_info *_get_rank_meta_info(pmix_common_dstore_ctx_t *ds_ctx, pmix_rank_t rank, + pmix_dstore_seg_desc_t *segdesc); +static uint8_t *_get_data_region_by_offset(pmix_common_dstore_ctx_t *ds_ctx, + pmix_dstore_seg_desc_t *segdesc, size_t offset); +static void _update_initial_segment_info(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map); +static void _set_constants_from_env(pmix_common_dstore_ctx_t *ds_ctx); +static inline ssize_t _get_univ_size(pmix_common_dstore_ctx_t *ds_ctx, const char *nspace); -static inline ns_map_data_t * _esh_session_map_search_server(const char *nspace); -static inline ns_map_data_t * _esh_session_map_search_client(const char *nspace); -static inline ns_map_data_t * _esh_session_map(const char *nspace, size_t tbl_idx); -static inline void _esh_session_map_clean(ns_map_t *m); -static inline int _esh_jobuid_tbl_search(uid_t jobuid, size_t *tbl_idx); -static inline int _esh_session_tbl_add(size_t *tbl_idx); -static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, int setjobuid); -static inline void _esh_session_release(session_t *s); -static inline void _esh_ns_track_cleanup(void); -static inline void _esh_sessions_cleanup(void); -static inline void _esh_ns_map_cleanup(void); +static inline ns_map_data_t * _esh_session_map_search_server(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace); +static inline ns_map_data_t * _esh_session_map_search_client(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace); +static inline ns_map_data_t * _esh_session_map(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, uint32_t local_size, + size_t tbl_idx); +static inline void _esh_session_map_clean(pmix_common_dstore_ctx_t *ds_ctx, ns_map_t *m); +static inline int _esh_jobuid_tbl_search(pmix_common_dstore_ctx_t *ds_ctx, + uid_t jobuid, size_t *tbl_idx); +static inline int _esh_session_tbl_add(pmix_common_dstore_ctx_t *ds_ctx, size_t *tbl_idx); +static int _esh_session_init(pmix_common_dstore_ctx_t *ds_ctx, size_t idx, ns_map_data_t *m, + uint32_t local_size, size_t jobuid, int setjobuid); +static void _esh_session_release(pmix_common_dstore_ctx_t *ds_ctx, size_t idx); +static inline void _esh_ns_track_cleanup(pmix_common_dstore_ctx_t *ds_ctx); +static inline void _esh_sessions_cleanup(pmix_common_dstore_ctx_t *ds_ctx); +static inline void _esh_ns_map_cleanup(pmix_common_dstore_ctx_t *ds_ctx); static inline int _esh_dir_del(const char *dirname); -static inline void _client_compat_save(pmix_peer_t *peer); -static inline pmix_peer_t * _client_peer(void); +static inline void _client_compat_save(pmix_common_dstore_ctx_t *ds_ctx, pmix_peer_t *peer); +static inline pmix_peer_t * _client_peer(pmix_common_dstore_ctx_t *ds_ctx); static inline int _my_client(const char *nspace, pmix_rank_t rank); -static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo); +static pmix_status_t _dstor_store_modex_cb(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_byte_object_t *bo); -static void dstore_finalize(void); +static pmix_status_t _dstore_store_nolock(pmix_common_dstore_ctx_t *ds_ctx, + ns_map_data_t *ns_map, + pmix_rank_t rank, + pmix_kval_t *kv); -static pmix_status_t dstore_setup_fork(const pmix_proc_t *peer, char ***env); - -static pmix_status_t dstore_cache_job_info(struct pmix_nspace_t *ns, - pmix_info_t info[], size_t ninfo); - -static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, - pmix_buffer_t *reply); - -static pmix_status_t dstore_store_job_info(const char *nspace, - pmix_buffer_t *job_data); - -static pmix_status_t _dstore_store(const char *nspace, - pmix_rank_t rank, - pmix_kval_t *kv); - -static pmix_status_t dstore_store(const pmix_proc_t *proc, - pmix_scope_t scope, - pmix_kval_t *kv); - -static pmix_status_t _dstore_fetch(const char *nspace, - pmix_rank_t rank, - const char *key, pmix_value_t **kvs); - -static pmix_status_t dstore_fetch(const pmix_proc_t *proc, - pmix_scope_t scope, bool copy, - const char *key, - pmix_info_t info[], size_t ninfo, - pmix_list_t *kvs); - -static pmix_status_t dstore_add_nspace(const char *nspace, - pmix_info_t info[], - size_t ninfo); - -static pmix_status_t dstore_del_nspace(const char* nspace); - -static pmix_status_t dstore_assign_module(pmix_info_t *info, size_t ninfo, - int *priority); - -static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace, - pmix_list_t *cbs, - pmix_byte_object_t *bo); - -pmix_gds_base_module_t pmix_ds12_module = { - .name = "ds12", - .init = dstore_init, - .finalize = dstore_finalize, - .assign_module = dstore_assign_module, - .cache_job_info = dstore_cache_job_info, - .register_job_info = dstore_register_job_info, - .store_job_info = dstore_store_job_info, - .store = dstore_store, - .store_modex = dstore_store_modex, - .fetch = dstore_fetch, - .setup_fork = dstore_setup_fork, - .add_nspace = dstore_add_nspace, - .del_nspace = dstore_del_nspace, -}; - -static char *_base_path = NULL; -static size_t _initial_segment_size = 0; -static size_t _max_ns_num; -static size_t _meta_segment_size = 0; -static size_t _max_meta_elems; -static size_t _data_segment_size = 0; -static size_t _lock_segment_size = 0; -static uid_t _jobuid; -static char _setjobuid = 0; -static pmix_peer_t *_clients_peer = NULL; - -static pmix_value_array_t *_session_array = NULL; -static pmix_value_array_t *_ns_map_array = NULL; -static pmix_value_array_t *_ns_track_array = NULL; +static pmix_status_t _dstore_fetch(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, pmix_rank_t rank, + const char *key, pmix_value_t **kvs); ns_map_data_t * (*_esh_session_map_search)(const char *nspace) = NULL; -int (*_esh_lock_init)(size_t idx) = NULL; -#define _ESH_SESSION_path(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].nspace_path) -#define _ESH_SESSION_lockfile(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].lockfile) -#define _ESH_SESSION_setjobuid(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].setjobuid) -#define _ESH_SESSION_jobuid(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].jobuid) -#define _ESH_SESSION_sm_seg_first(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].sm_seg_first) -#define _ESH_SESSION_sm_seg_last(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].sm_seg_last) -#define _ESH_SESSION_ns_info(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].ns_info) +#define _ESH_SESSION_lock(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].lock) + +#define _ESH_SESSION_path(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].nspace_path) + +#define _ESH_SESSION_lockfile(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].lockfile) + +#define _ESH_SESSION_setjobuid(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].setjobuid) + +#define _ESH_SESSION_jobuid(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].jobuid) + +#define _ESH_SESSION_sm_seg_first(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].sm_seg_first) +#define _ESH_SESSION_sm_seg_last(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].sm_seg_last) + +#define _ESH_SESSION_ns_info(session_array, tbl_idx) \ + (PMIX_VALUE_ARRAY_GET_BASE(session_array, session_t)[tbl_idx].ns_info) #ifdef ESH_PTHREAD_LOCK #define _ESH_SESSION_pthread_rwlock(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].rwlock) @@ -484,14 +157,13 @@ int (*_esh_lock_init)(size_t idx) = NULL; #define _ESH_SESSION_lock(tbl_idx) _ESH_SESSION_lockfd(tbl_idx) #endif -/* If _direct_mode is set, it means that we use linear search - * along the array of rank meta info objects inside a meta segment - * to find the requested rank. Otherwise, we do a fast lookup - * based on rank and directly compute offset. - * This mode is called direct because it's effectively used in - * sparse communication patterns when direct modex is usually used. - */ -static int _direct_mode = 0; +#define _ESH_LOCK(ds_ctx, session_id, operation) \ +__pmix_attribute_extension__ ({ \ + pmix_status_t rc = PMIX_SUCCESS; \ + rc = ds_ctx->lock_cbs->operation(_ESH_SESSION_lock(ds_ctx->session_array, \ + session_id)); \ + rc; \ +}) static void ncon(ns_track_elem_t *p) { memset(&p->ns_map, 0, sizeof(p->ns_map)); @@ -503,8 +175,8 @@ static void ncon(ns_track_elem_t *p) { } static void ndes(ns_track_elem_t *p) { - _delete_sm_desc(p->meta_seg); - _delete_sm_desc(p->data_seg); + pmix_common_dstor_delete_sm_desc(p->meta_seg); + pmix_common_dstor_delete_sm_desc(p->data_seg); memset(&p->ns_map, 0, sizeof(p->ns_map)); p->in_use = false; } @@ -513,153 +185,11 @@ PMIX_CLASS_INSTANCE(ns_track_elem_t, pmix_value_array_t, ncon, ndes); -static inline void _esh_session_map_clean(ns_map_t *m) { +static inline void _esh_session_map_clean(pmix_common_dstore_ctx_t *ds_ctx, ns_map_t *m) { memset(m, 0, sizeof(*m)); m->data.track_idx = -1; } -#ifdef ESH_FCNTL_LOCK -static inline int _flock_init(size_t idx) { - pmix_status_t rc = PMIX_SUCCESS; - - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - _ESH_SESSION_lock(idx) = open(_ESH_SESSION_lockfile(idx), O_CREAT | O_RDWR | O_EXCL, 0600); - - /* if previous launch was crashed, the lockfile might not be deleted and unlocked, - * so we delete it and create a new one. */ - if (_ESH_SESSION_lock(idx) < 0) { - unlink(_ESH_SESSION_lockfile(idx)); - _ESH_SESSION_lock(idx) = open(_ESH_SESSION_lockfile(idx), O_CREAT | O_RDWR, 0600); - if (_ESH_SESSION_lock(idx) < 0) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - } - if (_ESH_SESSION_setjobuid(idx) > 0) { - if (0 > chown(_ESH_SESSION_lockfile(idx), (uid_t) _ESH_SESSION_jobuid(idx), (gid_t) -1)) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - if (0 > chmod(_ESH_SESSION_lockfile(idx), S_IRUSR | S_IWGRP | S_IRGRP)) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - } - } - else { - _ESH_SESSION_lock(idx) = open(_ESH_SESSION_lockfile(idx), O_RDONLY); - if (-1 == _ESH_SESSION_lock(idx)) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - } - return rc; -} -#endif - -#ifdef ESH_PTHREAD_LOCK -static inline int _rwlock_init(size_t idx) { - pmix_status_t rc = PMIX_SUCCESS; - size_t size = _lock_segment_size; - pthread_rwlockattr_t attr; - - if ((NULL != _ESH_SESSION_pthread_seg(idx)) || (NULL != _ESH_SESSION_pthread_rwlock(idx))) { - rc = PMIX_ERR_INIT; - return rc; - } - _ESH_SESSION_pthread_seg(idx) = (pmix_pshmem_seg_t *)malloc(sizeof(pmix_pshmem_seg_t)); - if (NULL == _ESH_SESSION_pthread_seg(idx)) { - rc = PMIX_ERR_OUT_OF_RESOURCE; - return rc; - } - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_create(_ESH_SESSION_pthread_seg(idx), _ESH_SESSION_lockfile(idx), size))) { - return rc; - } - memset(_ESH_SESSION_pthread_seg(idx)->seg_base_addr, 0, size); - if (_ESH_SESSION_setjobuid(idx) > 0) { - if (0 > chown(_ESH_SESSION_lockfile(idx), (uid_t) _ESH_SESSION_jobuid(idx), (gid_t) -1)){ - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - /* set the mode as required */ - if (0 > chmod(_ESH_SESSION_lockfile(idx), S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP )) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - } - _ESH_SESSION_pthread_rwlock(idx) = (pthread_rwlock_t *)_ESH_SESSION_pthread_seg(idx)->seg_base_addr; - - if (0 != pthread_rwlockattr_init(&attr)) { - rc = PMIX_ERR_INIT; - pmix_pshmem.segment_detach(_ESH_SESSION_pthread_seg(idx)); - return rc; - } - if (0 != pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { - rc = PMIX_ERR_INIT; - pmix_pshmem.segment_detach(_ESH_SESSION_pthread_seg(idx)); - pthread_rwlockattr_destroy(&attr); - return rc; - } -#ifdef HAVE_PTHREAD_SETKIND - if (0 != pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)) { - rc = PMIX_ERR_INIT; - pmix_pshmem.segment_detach(_ESH_SESSION_pthread_seg(idx)); - pthread_rwlockattr_destroy(&attr); - return rc; - } -#endif - if (0 != pthread_rwlock_init(_ESH_SESSION_pthread_rwlock(idx), &attr)) { - rc = PMIX_ERR_INIT; - pmix_pshmem.segment_detach(_ESH_SESSION_pthread_seg(idx)); - pthread_rwlockattr_destroy(&attr); - return rc; - } - if (0 != pthread_rwlockattr_destroy(&attr)) { - rc = PMIX_ERR_INIT; - return rc; - } - - } - else { - _ESH_SESSION_pthread_seg(idx)->seg_size = size; - snprintf(_ESH_SESSION_pthread_seg(idx)->seg_name, PMIX_PATH_MAX, "%s", _ESH_SESSION_lockfile(idx)); - if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_attach(_ESH_SESSION_pthread_seg(idx), PMIX_PSHMEM_RW))) { - return rc; - } - _ESH_SESSION_pthread_rwlock(idx) = (pthread_rwlock_t *)_ESH_SESSION_pthread_seg(idx)->seg_base_addr; - } - - return rc; -} - -static inline void _rwlock_release(session_t *s) { - pmix_status_t rc; - - if (0 != pthread_rwlock_destroy(s->rwlock)) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return; - } - - /* detach & unlink from current desc */ - if (s->rwlock_seg->seg_cpid == getpid()) { - pmix_pshmem.segment_unlink(s->rwlock_seg); - } - pmix_pshmem.segment_detach(s->rwlock_seg); - - free(s->rwlock_seg); - s->rwlock_seg = NULL; - s->rwlock = NULL; -} -#endif - static inline int _esh_dir_del(const char *path) { DIR *dir; @@ -715,132 +245,133 @@ static inline int _esh_dir_del(const char *path) return rc; } -static inline int _esh_tbls_init(void) +static inline int _esh_tbls_init(pmix_common_dstore_ctx_t *ds_ctx) { pmix_status_t rc = PMIX_SUCCESS; size_t idx; /* initial settings */ - _ns_track_array = NULL; - _session_array = NULL; - _ns_map_array = NULL; + ds_ctx->ns_track_array = NULL; + ds_ctx->session_array = NULL; + ds_ctx->ns_map_array = NULL; /* Setup namespace tracking array */ - if (NULL == (_ns_track_array = PMIX_NEW(pmix_value_array_t))) { + if (NULL == (ds_ctx->ns_track_array = PMIX_NEW(pmix_value_array_t))) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_init(_ns_track_array, sizeof(ns_track_elem_t)))){ + if (PMIX_SUCCESS != (rc = pmix_value_array_init(ds_ctx->ns_track_array, sizeof(ns_track_elem_t)))){ PMIX_ERROR_LOG(rc); goto err_exit; } /* Setup sessions table */ - if (NULL == (_session_array = PMIX_NEW(pmix_value_array_t))){ + if (NULL == (ds_ctx->session_array = PMIX_NEW(pmix_value_array_t))){ rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_init(_session_array, sizeof(session_t)))) { + if (PMIX_SUCCESS != (rc = pmix_value_array_init(ds_ctx->session_array, sizeof(session_t)))) { PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(_session_array, ESH_INIT_SESSION_TBL_SIZE))) { + if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(ds_ctx->session_array, ESH_INIT_SESSION_TBL_SIZE))) { PMIX_ERROR_LOG(rc); goto err_exit; } for (idx = 0; idx < ESH_INIT_SESSION_TBL_SIZE; idx++) { - memset(pmix_value_array_get_item(_session_array, idx), 0, sizeof(session_t)); + memset(pmix_value_array_get_item(ds_ctx->session_array, idx), 0, sizeof(session_t)); } /* Setup namespace map array */ - if (NULL == (_ns_map_array = PMIX_NEW(pmix_value_array_t))) { + if (NULL == (ds_ctx->ns_map_array = PMIX_NEW(pmix_value_array_t))) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_init(_ns_map_array, sizeof(ns_map_t)))) { + if (PMIX_SUCCESS != (rc = pmix_value_array_init(ds_ctx->ns_map_array, sizeof(ns_map_t)))) { PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(_ns_map_array, ESH_INIT_NS_MAP_TBL_SIZE))) { + if (PMIX_SUCCESS != (rc = pmix_value_array_set_size(ds_ctx->ns_map_array, ESH_INIT_NS_MAP_TBL_SIZE))) { PMIX_ERROR_LOG(rc); goto err_exit; } for (idx = 0; idx < ESH_INIT_NS_MAP_TBL_SIZE; idx++) { - _esh_session_map_clean(pmix_value_array_get_item(_ns_map_array, idx)); + _esh_session_map_clean(ds_ctx, pmix_value_array_get_item(ds_ctx->ns_map_array, idx)); } return PMIX_SUCCESS; err_exit: - if (NULL != _ns_track_array) { - PMIX_RELEASE(_ns_track_array); + if (NULL != ds_ctx->ns_track_array) { + PMIX_RELEASE(ds_ctx->ns_track_array); } - if (NULL != _session_array) { - PMIX_RELEASE(_session_array); + if (NULL != ds_ctx->session_array) { + PMIX_RELEASE(ds_ctx->session_array); } - if (NULL != _ns_map_array) { - PMIX_RELEASE(_ns_map_array); + if (NULL != ds_ctx->ns_map_array) { + PMIX_RELEASE(ds_ctx->ns_map_array); } return rc; } -static inline void _esh_ns_map_cleanup(void) +static inline void _esh_ns_map_cleanup(pmix_common_dstore_ctx_t *ds_ctx) { size_t idx; size_t size; ns_map_t *ns_map; - if (NULL == _ns_map_array) { + if (NULL == ds_ctx->ns_map_array) { return; } - size = pmix_value_array_get_size(_ns_map_array); - ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t); + size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); for (idx = 0; idx < size; idx++) { - if(ns_map[idx].in_use) - _esh_session_map_clean(&ns_map[idx]); + if(ns_map[idx].in_use) { + _esh_session_map_clean(ds_ctx, &ns_map[idx]); + } } - PMIX_RELEASE(_ns_map_array); - _ns_map_array = NULL; + PMIX_RELEASE(ds_ctx->ns_map_array); + ds_ctx->ns_map_array = NULL; } -static inline void _esh_sessions_cleanup(void) +static inline void _esh_sessions_cleanup(pmix_common_dstore_ctx_t *ds_ctx) { size_t idx; size_t size; session_t *s_tbl; - if (NULL == _session_array) { + if (NULL == ds_ctx->session_array) { return; } - size = pmix_value_array_get_size(_session_array); - s_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t); + size = pmix_value_array_get_size(ds_ctx->session_array); + s_tbl = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->session_array, session_t); for (idx = 0; idx < size; idx++) { if(s_tbl[idx].in_use) - _esh_session_release(&s_tbl[idx]); + _esh_session_release(ds_ctx, idx); } - PMIX_RELEASE(_session_array); - _session_array = NULL; + PMIX_RELEASE(ds_ctx->session_array); + ds_ctx->session_array = NULL; } -static inline void _esh_ns_track_cleanup(void) +static inline void _esh_ns_track_cleanup(pmix_common_dstore_ctx_t *ds_ctx) { int size; ns_track_elem_t *ns_trk; - if (NULL == _ns_track_array) { + if (NULL == ds_ctx->ns_track_array) { return; } - size = pmix_value_array_get_size(_ns_track_array); - ns_trk = PMIX_VALUE_ARRAY_GET_BASE(_ns_track_array, ns_track_elem_t); + size = pmix_value_array_get_size(ds_ctx->ns_track_array); + ns_trk = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_track_array, ns_track_elem_t); for (int i = 0; i < size; i++) { ns_track_elem_t *trk = ns_trk + i; @@ -849,15 +380,17 @@ static inline void _esh_ns_track_cleanup(void) } } - PMIX_RELEASE(_ns_track_array); - _ns_track_array = NULL; + PMIX_RELEASE(ds_ctx->ns_track_array); + ds_ctx->ns_track_array = NULL; } -static inline ns_map_data_t * _esh_session_map(const char *nspace, size_t tbl_idx) +static inline ns_map_data_t * _esh_session_map(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, uint32_t local_size, + size_t tbl_idx) { size_t map_idx; - size_t size = pmix_value_array_get_size(_ns_map_array);; - ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t);; + size_t size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); ns_map_t *new_map = NULL; if (NULL == nspace) { @@ -868,32 +401,33 @@ static inline ns_map_data_t * _esh_session_map(const char *nspace, size_t tbl_id for(map_idx = 0; map_idx < size; map_idx++) { if (!ns_map[map_idx].in_use) { ns_map[map_idx].in_use = true; - strncpy(ns_map[map_idx].data.name, nspace, sizeof(ns_map[map_idx].data.name)-1); + pmix_strncpy(ns_map[map_idx].data.name, nspace, sizeof(ns_map[map_idx].data.name)-1); ns_map[map_idx].data.tbl_idx = tbl_idx; return &ns_map[map_idx].data; } } - if (NULL == (new_map = pmix_value_array_get_item(_ns_map_array, map_idx))) { + if (NULL == (new_map = pmix_value_array_get_item(ds_ctx->ns_map_array, map_idx))) { PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); return NULL; } - _esh_session_map_clean(new_map); + _esh_session_map_clean(ds_ctx, new_map); new_map->in_use = true; new_map->data.tbl_idx = tbl_idx; - strncpy(new_map->data.name, nspace, sizeof(new_map->data.name)-1); + pmix_strncpy(new_map->data.name, nspace, sizeof(new_map->data.name)-1); return &new_map->data; } -static inline int _esh_jobuid_tbl_search(uid_t jobuid, size_t *tbl_idx) +static inline int _esh_jobuid_tbl_search(pmix_common_dstore_ctx_t *ds_ctx, + uid_t jobuid, size_t *tbl_idx) { size_t idx, size; session_t *session_tbl = NULL; - size = pmix_value_array_get_size(_session_array); - session_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t); + size = pmix_value_array_get_size(ds_ctx->session_array); + session_tbl = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->session_array, session_t); for(idx = 0; idx < size; idx++) { if (session_tbl[idx].in_use && session_tbl[idx].jobuid == jobuid) { @@ -905,37 +439,38 @@ static inline int _esh_jobuid_tbl_search(uid_t jobuid, size_t *tbl_idx) return PMIX_ERR_NOT_FOUND; } -static inline int _esh_session_tbl_add(size_t *tbl_idx) +static inline int _esh_session_tbl_add(pmix_common_dstore_ctx_t *ds_ctx, size_t *tbl_idx) { size_t idx; - size_t size = pmix_value_array_get_size(_session_array); - session_t *s_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t); + size_t size = pmix_value_array_get_size(ds_ctx->session_array); + session_t *s_tbl = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->session_array, session_t); session_t *new_sesion; pmix_status_t rc = PMIX_SUCCESS; for(idx = 0; idx < size; idx ++) { if (0 == s_tbl[idx].in_use) { - s_tbl[idx].in_use = 1; - *tbl_idx = idx; - return PMIX_SUCCESS; + goto done; } } - if (NULL == (new_sesion = pmix_value_array_get_item(_session_array, idx))) { + if (NULL == (new_sesion = pmix_value_array_get_item(ds_ctx->session_array, idx))) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); return rc; } + +done: s_tbl[idx].in_use = 1; *tbl_idx = idx; return PMIX_SUCCESS; } -static inline ns_map_data_t * _esh_session_map_search_server(const char *nspace) +static inline ns_map_data_t * _esh_session_map_search_server(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace) { - size_t idx, size = pmix_value_array_get_size(_ns_map_array); - ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t); + size_t idx, size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); if (NULL == nspace) { return NULL; } @@ -949,10 +484,11 @@ static inline ns_map_data_t * _esh_session_map_search_server(const char *nspace) return NULL; } -static inline ns_map_data_t * _esh_session_map_search_client(const char *nspace) +static inline ns_map_data_t * _esh_session_map_search_client(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace) { - size_t idx, size = pmix_value_array_get_size(_ns_map_array); - ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t); + size_t idx, size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map_t *ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); if (NULL == nspace) { return NULL; @@ -964,29 +500,19 @@ static inline ns_map_data_t * _esh_session_map_search_client(const char *nspace) return &ns_map[idx].data; } } - return _esh_session_map(nspace, 0); + return _esh_session_map(ds_ctx, nspace, 0, 0); } -static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, int setjobuid) +static int _esh_session_init(pmix_common_dstore_ctx_t *ds_ctx, size_t idx, ns_map_data_t *m, + uint32_t local_size, size_t jobuid, int setjobuid) { - seg_desc_t *seg = NULL; - session_t *s = &(PMIX_VALUE_ARRAY_GET_ITEM(_session_array, session_t, idx)); + pmix_dstore_seg_desc_t *seg = NULL; + session_t *s = &(PMIX_VALUE_ARRAY_GET_ITEM(ds_ctx->session_array, session_t, idx)); pmix_status_t rc = PMIX_SUCCESS; s->setjobuid = setjobuid; s->jobuid = jobuid; - s->nspace_path = strdup(_base_path); - - /* create a lock file to prevent clients from reading while server is writing to the shared memory. - * This situation is quite often, especially in case of direct modex when clients might ask for data - * simultaneously.*/ - if(0 > asprintf(&s->lockfile, "%s/dstore_sm.lock", s->nspace_path)) { - rc = PMIX_ERR_OUT_OF_RESOURCE; - PMIX_ERROR_LOG(rc); - return rc; - } - PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, - "%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, s->lockfile)); + s->nspace_path = strdup(ds_ctx->base_path); if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { if (0 != mkdir(s->nspace_path, 0770)) { @@ -1005,7 +531,8 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, return rc; } } - seg = _create_new_segment(INITIAL_SEGMENT, m, 0); + seg = pmix_common_dstor_create_new_segment(PMIX_DSTORE_INITIAL_SEGMENT, ds_ctx->base_path, + m->name, 0, ds_ctx->jobuid, ds_ctx->setjobuid); if( NULL == seg ){ rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); @@ -1013,245 +540,92 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, } } else { - seg = _attach_new_segment(INITIAL_SEGMENT, m, 0); + seg = pmix_common_dstor_attach_new_segment(PMIX_DSTORE_INITIAL_SEGMENT, ds_ctx->base_path, m->name, 0); if( NULL == seg ){ rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); return rc; } } - - if (NULL == _esh_lock_init) { - rc = PMIX_ERR_INIT; - PMIX_ERROR_LOG(rc); - return rc; - } - if ( PMIX_SUCCESS != (rc = _esh_lock_init(m->tbl_idx))) { - PMIX_ERROR_LOG(rc); - return rc; - } - s->sm_seg_first = seg; s->sm_seg_last = s->sm_seg_first; + return PMIX_SUCCESS; } -static inline void _esh_session_release(session_t *s) +static void _esh_session_release(pmix_common_dstore_ctx_t *ds_ctx, size_t idx) { + session_t *s = &(PMIX_VALUE_ARRAY_GET_ITEM(ds_ctx->session_array, session_t, idx)); + if (!s->in_use) { return; } - _delete_sm_desc(s->sm_seg_first); - /* if the lock fd was somehow set, then we - * need to close it */ - if (0 != s->lockfd) { - close(s->lockfd); - } + pmix_common_dstor_delete_sm_desc(s->sm_seg_first); + + ds_ctx->lock_cbs->finalize(&_ESH_SESSION_lock(ds_ctx->session_array, idx)); - if (NULL != s->lockfile) { - if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - unlink(s->lockfile); - } - free(s->lockfile); - } if (NULL != s->nspace_path) { if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { _esh_dir_del(s->nspace_path); } free(s->nspace_path); } -#ifdef ESH_PTHREAD_LOCK - _rwlock_release(s); -#endif memset ((char *) s, 0, sizeof(*s)); } -static void _set_constants_from_env() +static void _set_constants_from_env(pmix_common_dstore_ctx_t *ds_ctx) { char *str; - int page_size = _pmix_getpagesize(); + int page_size = pmix_common_dstor_getpagesize(); if( NULL != (str = getenv(ESH_ENV_INITIAL_SEG_SIZE)) ) { - _initial_segment_size = strtoul(str, NULL, 10); - if ((size_t)page_size > _initial_segment_size) { - _initial_segment_size = (size_t)page_size; + ds_ctx->initial_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > ds_ctx->initial_segment_size) { + ds_ctx->initial_segment_size = (size_t)page_size; } } - if (0 == _initial_segment_size) { - _initial_segment_size = INITIAL_SEG_SIZE; + if (0 == ds_ctx->initial_segment_size) { + ds_ctx->initial_segment_size = INITIAL_SEG_SIZE; } if( NULL != (str = getenv(ESH_ENV_NS_META_SEG_SIZE)) ) { - _meta_segment_size = strtoul(str, NULL, 10); - if ((size_t)page_size > _meta_segment_size) { - _meta_segment_size = (size_t)page_size; + ds_ctx->meta_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > ds_ctx->meta_segment_size) { + ds_ctx->meta_segment_size = (size_t)page_size; } } - if (0 == _meta_segment_size) { - _meta_segment_size = NS_META_SEG_SIZE; + if (0 == ds_ctx->meta_segment_size) { + ds_ctx->meta_segment_size = NS_META_SEG_SIZE; } if( NULL != (str = getenv(ESH_ENV_NS_DATA_SEG_SIZE)) ) { - _data_segment_size = strtoul(str, NULL, 10); - if ((size_t)page_size > _data_segment_size) { - _data_segment_size = (size_t)page_size; + ds_ctx->data_segment_size = strtoul(str, NULL, 10); + if ((size_t)page_size > ds_ctx->data_segment_size) { + ds_ctx->data_segment_size = (size_t)page_size; } } - if (0 == _data_segment_size) { - _data_segment_size = NS_DATA_SEG_SIZE; + if (0 == ds_ctx->data_segment_size) { + ds_ctx->data_segment_size = NS_DATA_SEG_SIZE; } if (NULL != (str = getenv(ESH_ENV_LINEAR))) { if (1 == strtoul(str, NULL, 10)) { - _direct_mode = 1; + ds_ctx->direct_mode = 1; } } - _lock_segment_size = page_size; - _max_ns_num = (_initial_segment_size - sizeof(size_t) * 2) / sizeof(ns_seg_info_t); - _max_meta_elems = (_meta_segment_size - sizeof(size_t)) / sizeof(rank_meta_info); + ds_ctx->lock_segment_size = page_size; + ds_ctx->max_ns_num = (ds_ctx->initial_segment_size - sizeof(size_t) * 2) / sizeof(ns_seg_info_t); + ds_ctx->max_meta_elems = (ds_ctx->meta_segment_size - sizeof(size_t)) / sizeof(rank_meta_info); -} + pmix_common_dstor_init_segment_info(ds_ctx->initial_segment_size, ds_ctx->meta_segment_size, + ds_ctx->data_segment_size); -static void _delete_sm_desc(seg_desc_t *desc) -{ - seg_desc_t *tmp; - - /* free all global segments */ - while (NULL != desc) { - tmp = desc->next; - /* detach & unlink from current desc */ - if (desc->seg_info.seg_cpid == getpid()) { - pmix_pshmem.segment_unlink(&desc->seg_info); - } - pmix_pshmem.segment_detach(&desc->seg_info); - free(desc); - desc = tmp; - } -} - -static int _pmix_getpagesize(void) -{ -#if defined(_SC_PAGESIZE ) - return sysconf(_SC_PAGESIZE); -#elif defined(_SC_PAGE_SIZE) - return sysconf(_SC_PAGE_SIZE); -#else - return 65536; /* safer to overestimate than under */ -#endif -} - -static seg_desc_t *_create_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id) -{ - pmix_status_t rc; - char file_name[PMIX_PATH_MAX]; - size_t size; - seg_desc_t *new_seg = NULL; - - PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, - "%s:%d:%s: segment type %d, nspace %s, id %u", - __FILE__, __LINE__, __func__, type, ns_map->name, id)); - - switch (type) { - case INITIAL_SEGMENT: - size = _initial_segment_size; - snprintf(file_name, PMIX_PATH_MAX, "%s/initial-pmix_shared-segment-%u", - _ESH_SESSION_path(ns_map->tbl_idx), id); - break; - case NS_META_SEGMENT: - size = _meta_segment_size; - snprintf(file_name, PMIX_PATH_MAX, "%s/smseg-%s-%u", - _ESH_SESSION_path(ns_map->tbl_idx), ns_map->name, id); - break; - case NS_DATA_SEGMENT: - size = _data_segment_size; - snprintf(file_name, PMIX_PATH_MAX, "%s/smdataseg-%s-%d", - _ESH_SESSION_path(ns_map->tbl_idx), ns_map->name, id); - break; - default: - PMIX_ERROR_LOG(PMIX_ERROR); - return NULL; - } - new_seg = (seg_desc_t*)malloc(sizeof(seg_desc_t)); - if (new_seg) { - new_seg->id = id; - new_seg->next = NULL; - new_seg->type = type; - rc = pmix_pshmem.segment_create(&new_seg->seg_info, file_name, size); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - goto err_exit; - } - memset(new_seg->seg_info.seg_base_addr, 0, size); - - - if (_ESH_SESSION_setjobuid(ns_map->tbl_idx) > 0){ - rc = PMIX_ERR_PERM; - if (0 > chown(file_name, (uid_t) _ESH_SESSION_jobuid(ns_map->tbl_idx), (gid_t) -1)){ - PMIX_ERROR_LOG(rc); - goto err_exit; - } - /* set the mode as required */ - if (0 > chmod(file_name, S_IRUSR | S_IRGRP | S_IWGRP )) { - PMIX_ERROR_LOG(rc); - goto err_exit; - } - } - } - return new_seg; - -err_exit: - if( NULL != new_seg ){ - free(new_seg); - } - return NULL; -} - -static seg_desc_t *_attach_new_segment(segment_type type, const ns_map_data_t *ns_map, uint32_t id) -{ - pmix_status_t rc; - seg_desc_t *new_seg = NULL; - new_seg = (seg_desc_t*)malloc(sizeof(seg_desc_t)); - new_seg->id = id; - new_seg->next = NULL; - new_seg->type = type; - - PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, - "%s:%d:%s: segment type %d, nspace %s, id %u", - __FILE__, __LINE__, __func__, type, ns_map->name, id)); - - switch (type) { - case INITIAL_SEGMENT: - new_seg->seg_info.seg_size = _initial_segment_size; - snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/initial-pmix_shared-segment-%u", - _ESH_SESSION_path(ns_map->tbl_idx), id); - break; - case NS_META_SEGMENT: - new_seg->seg_info.seg_size = _meta_segment_size; - snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smseg-%s-%u", - _ESH_SESSION_path(ns_map->tbl_idx), ns_map->name, id); - break; - case NS_DATA_SEGMENT: - new_seg->seg_info.seg_size = _data_segment_size; - snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smdataseg-%s-%d", - _ESH_SESSION_path(ns_map->tbl_idx), ns_map->name, id); - break; - default: - free(new_seg); - PMIX_ERROR_LOG(PMIX_ERROR); - return NULL; - } - rc = pmix_pshmem.segment_attach(&new_seg->seg_info, PMIX_PSHMEM_RONLY); - if (PMIX_SUCCESS != rc) { - free(new_seg); - new_seg = NULL; - PMIX_ERROR_LOG(rc); - } - return new_seg; } /* This function synchronizes the content of initial shared segment and the local track list. */ -static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) +static int _update_ns_elem(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_elem, + ns_seg_info_t *info) { - seg_desc_t *seg, *tmp = NULL; + pmix_dstore_seg_desc_t *seg, *tmp = NULL; size_t i, offs; ns_map_data_t *ns_map = NULL; pmix_status_t rc; @@ -1260,7 +634,7 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) "%s:%d:%s", __FILE__, __LINE__, __func__)); - if (NULL == (ns_map = _esh_session_map_search(info->ns_map.name))) { + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, info->ns_map.name))) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); return rc; @@ -1276,14 +650,16 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) /* synchronize number of meta segments for the target namespace. */ for (i = ns_elem->num_meta_seg; i < info->num_meta_seg; i++) { if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - seg = _create_new_segment(NS_META_SEGMENT, &info->ns_map, i); + seg = pmix_common_dstor_create_new_segment(PMIX_DSTORE_NS_META_SEGMENT, ds_ctx->base_path, + info->ns_map.name, i, ds_ctx->jobuid, + ds_ctx->setjobuid); if (NULL == seg) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); return rc; } } else { - seg = _attach_new_segment(NS_META_SEGMENT, &info->ns_map, i); + seg = pmix_common_dstor_attach_new_segment(PMIX_DSTORE_NS_META_SEGMENT, ds_ctx->base_path, info->ns_map.name, i); if (NULL == seg) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); @@ -1309,7 +685,9 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) /* synchronize number of data segments for the target namespace. */ for (i = ns_elem->num_data_seg; i < info->num_data_seg; i++) { if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - seg = _create_new_segment(NS_DATA_SEGMENT, &info->ns_map, i); + seg = pmix_common_dstor_create_new_segment(PMIX_DSTORE_NS_DATA_SEGMENT, ds_ctx->base_path, + info->ns_map.name, i, ds_ctx->jobuid, + ds_ctx->setjobuid); if (NULL == seg) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); @@ -1318,7 +696,7 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) offs = sizeof(size_t);//shift on offset field itself memcpy(seg->seg_info.seg_base_addr, &offs, sizeof(size_t)); } else { - seg = _attach_new_segment(NS_DATA_SEGMENT, &info->ns_map, i); + seg = pmix_common_dstor_attach_new_segment(PMIX_DSTORE_NS_DATA_SEGMENT, ds_ctx->base_path, info->ns_map.name, i); if (NULL == seg) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); @@ -1338,66 +716,54 @@ static int _update_ns_elem(ns_track_elem_t *ns_elem, ns_seg_info_t *info) return PMIX_SUCCESS; } -static seg_desc_t *extend_segment(seg_desc_t *segdesc, const ns_map_data_t *ns_map) -{ - seg_desc_t *tmp, *seg; - - PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, - "%s:%d:%s", - __FILE__, __LINE__, __func__)); - /* find last segment */ - tmp = segdesc; - while (NULL != tmp->next) { - tmp = tmp->next; - } - /* create another segment, the old one is full. */ - seg = _create_new_segment(segdesc->type, ns_map, tmp->id + 1); - tmp->next = seg; - - return seg; -} - -static int _put_ns_info_to_initial_segment(const ns_map_data_t *ns_map, pmix_pshmem_seg_t *metaseg, pmix_pshmem_seg_t *dataseg) +static int _put_ns_info_to_initial_segment(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map, pmix_pshmem_seg_t *metaseg, + pmix_pshmem_seg_t *dataseg) { ns_seg_info_t elem; size_t num_elems; - num_elems = *((size_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr)); - seg_desc_t *last_seg = _ESH_SESSION_sm_seg_last(ns_map->tbl_idx); + num_elems = *((size_t*)(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, + ns_map->tbl_idx)->seg_info.seg_base_addr)); + pmix_dstore_seg_desc_t *last_seg = _ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx); pmix_status_t rc; PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s", __FILE__, __LINE__, __func__)); - if (_max_ns_num == num_elems) { + if (ds_ctx->max_ns_num == num_elems) { num_elems = 0; - if (NULL == (last_seg = extend_segment(last_seg, ns_map))) { + if (NULL == (last_seg = pmix_common_dstor_extend_segment(last_seg, ds_ctx->base_path, ns_map->name, + ds_ctx->jobuid, ds_ctx->setjobuid))) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); return rc; } /* mark previous segment as full */ size_t full = 1; - memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr + sizeof(size_t)), &full, sizeof(size_t)); - _ESH_SESSION_sm_seg_last(ns_map->tbl_idx) = last_seg; - memset(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr, 0, _initial_segment_size); + memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx)->seg_info.seg_base_addr + + sizeof(size_t)), &full, sizeof(size_t)); + _ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx) = last_seg; + memset(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx)->seg_info.seg_base_addr, + 0, ds_ctx->initial_segment_size); } memset(&elem.ns_map, 0, sizeof(elem.ns_map)); - strncpy(elem.ns_map.name, ns_map->name, sizeof(elem.ns_map.name)-1); + pmix_strncpy(elem.ns_map.name, ns_map->name, sizeof(elem.ns_map.name)-1); elem.ns_map.tbl_idx = ns_map->tbl_idx; elem.num_meta_seg = 1; elem.num_data_seg = 1; - memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr) + sizeof(size_t) * 2 + num_elems * sizeof(ns_seg_info_t), - &elem, sizeof(ns_seg_info_t)); + memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx)->seg_info.seg_base_addr) + + sizeof(size_t) * 2 + num_elems * sizeof(ns_seg_info_t), &elem, sizeof(ns_seg_info_t)); num_elems++; - memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ns_map->tbl_idx)->seg_info.seg_base_addr), &num_elems, sizeof(size_t)); + memcpy((uint8_t*)(_ESH_SESSION_sm_seg_last(ds_ctx->session_array, ns_map->tbl_idx)->seg_info.seg_base_addr), + &num_elems, sizeof(size_t)); return PMIX_SUCCESS; } /* clients should sync local info with information from initial segment regularly */ -static void _update_initial_segment_info(const ns_map_data_t *ns_map) +static void _update_initial_segment_info(pmix_common_dstore_ctx_t *ds_ctx, const ns_map_data_t *ns_map) { - seg_desc_t *tmp; - tmp = _ESH_SESSION_sm_seg_first(ns_map->tbl_idx); + pmix_dstore_seg_desc_t *tmp; + tmp = _ESH_SESSION_sm_seg_first(ds_ctx->session_array, ns_map->tbl_idx); PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, "%s:%d:%s", __FILE__, __LINE__, __func__)); @@ -1406,7 +772,8 @@ static void _update_initial_segment_info(const ns_map_data_t *ns_map) do { /* check if current segment was marked as full but no more next segment is in the chain */ if (NULL == tmp->next && 1 == *((size_t*)((uint8_t*)(tmp->seg_info.seg_base_addr) + sizeof(size_t)))) { - tmp->next = _attach_new_segment(INITIAL_SEGMENT, ns_map, tmp->id+1); + tmp->next = pmix_common_dstor_attach_new_segment(PMIX_DSTORE_INITIAL_SEGMENT, ds_ctx->base_path, + ns_map->name, tmp->id+1); } tmp = tmp->next; } @@ -1414,11 +781,12 @@ static void _update_initial_segment_info(const ns_map_data_t *ns_map) } /* this function will be used by clients to get ns data from the initial segment and add them to the tracker list */ -static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_map) +static ns_seg_info_t *_get_ns_info_from_initial_segment(pmix_common_dstore_ctx_t *ds_ctx, + const ns_map_data_t *ns_map) { pmix_status_t rc; size_t i; - seg_desc_t *tmp; + pmix_dstore_seg_desc_t *tmp; ns_seg_info_t *elem, *cur_elem; elem = NULL; size_t num_elems; @@ -1426,7 +794,7 @@ static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_ PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, "%s:%d:%s", __FILE__, __LINE__, __func__)); - tmp = _ESH_SESSION_sm_seg_first(ns_map->tbl_idx); + tmp = _ESH_SESSION_sm_seg_first(ds_ctx->session_array, ns_map->tbl_idx); rc = 1; /* go through all global segments */ @@ -1448,10 +816,11 @@ static ns_seg_info_t *_get_ns_info_from_initial_segment(const ns_map_data_t *ns_ return elem; } -static ns_track_elem_t *_get_track_elem_for_namespace(ns_map_data_t *ns_map) +static ns_track_elem_t *_get_track_elem_for_namespace(pmix_common_dstore_ctx_t *ds_ctx, + ns_map_data_t *ns_map) { ns_track_elem_t *new_elem = NULL; - size_t size = pmix_value_array_get_size(_ns_track_array); + size_t size = pmix_value_array_get_size(ds_ctx->ns_track_array); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: nspace %s", @@ -1464,27 +833,27 @@ static ns_track_elem_t *_get_track_elem_for_namespace(ns_map_data_t *ns_map) } /* data for this namespace should be already stored in shared memory region. */ /* so go and just put new data. */ - return pmix_value_array_get_item(_ns_track_array, ns_map->track_idx); + return pmix_value_array_get_item(ds_ctx->ns_track_array, ns_map->track_idx); } /* create shared memory regions for this namespace and store its info locally * to operate with address and detach/unlink afterwards. */ - if (NULL == (new_elem = pmix_value_array_get_item(_ns_track_array, size))) { + if (NULL == (new_elem = pmix_value_array_get_item(ds_ctx->ns_track_array, size))) { return NULL; } PMIX_CONSTRUCT(new_elem, ns_track_elem_t); - strncpy(new_elem->ns_map.name, ns_map->name, sizeof(new_elem->ns_map.name)-1); + pmix_strncpy(new_elem->ns_map.name, ns_map->name, sizeof(new_elem->ns_map.name)-1); /* save latest track idx to info of nspace */ ns_map->track_idx = size; return new_elem; } -static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc) +static rank_meta_info *_get_rank_meta_info(pmix_common_dstore_ctx_t *ds_ctx, pmix_rank_t rank, pmix_dstore_seg_desc_t *segdesc) { size_t i; rank_meta_info *elem = NULL; - seg_desc_t *tmp = segdesc; + pmix_dstore_seg_desc_t *tmp = segdesc; size_t num_elems, rel_offset; int id; rank_meta_info *cur_elem; @@ -1495,7 +864,7 @@ static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc "%s:%d:%s", __FILE__, __LINE__, __func__)); - if (1 == _direct_mode) { + if (1 == ds_ctx->direct_mode) { /* do linear search to find the requested rank inside all meta segments * for this namespace. */ /* go through all existing meta segments for this namespace */ @@ -1514,8 +883,8 @@ static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc } else { /* directly compute index of meta segment (id) and relative offset (rel_offset) * inside this segment for fast lookup a rank_meta_info object for the requested rank. */ - id = rcount/_max_meta_elems; - rel_offset = (rcount%_max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); + id = rcount/ds_ctx->max_meta_elems; + rel_offset = (rcount % ds_ctx->max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); /* go through all existing meta segments for this namespace. * Stop at id number if it exists. */ while (NULL != tmp->next && 0 != id) { @@ -1534,10 +903,10 @@ static rank_meta_info *_get_rank_meta_info(pmix_rank_t rank, seg_desc_t *segdesc return elem; } -static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) +static int set_rank_meta_info(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, rank_meta_info *rinfo) { /* it's claimed that there is still no meta info for this rank stored */ - seg_desc_t *tmp; + pmix_dstore_seg_desc_t *tmp; size_t num_elems, rel_offset; int id, count; rank_meta_info *cur_elem; @@ -1554,18 +923,19 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) (unsigned long)rinfo->offset, (unsigned long)rinfo->count)); tmp = ns_info->meta_seg; - if (1 == _direct_mode) { + if (1 == ds_ctx->direct_mode) { /* get the last meta segment to put new rank_meta_info at the end. */ while (NULL != tmp->next) { tmp = tmp->next; } num_elems = *((size_t*)(tmp->seg_info.seg_base_addr)); - if (_max_meta_elems <= num_elems) { + if (ds_ctx->max_meta_elems <= num_elems) { PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, "%s:%d:%s: extend meta segment for nspace %s", __FILE__, __LINE__, __func__, ns_info->ns_map.name)); /* extend meta segment, so create a new one */ - tmp = extend_segment(tmp, &ns_info->ns_map); + tmp = pmix_common_dstor_extend_segment(tmp, ds_ctx->base_path, ns_info->ns_map.name, + ds_ctx->jobuid, ds_ctx->setjobuid); if (NULL == tmp) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; @@ -1573,7 +943,7 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) ns_info->num_meta_seg++; memset(tmp->seg_info.seg_base_addr, 0, sizeof(rank_meta_info)); /* update number of meta segments for namespace in initial_segment */ - ns_seg_info_t *elem = _get_ns_info_from_initial_segment(&ns_info->ns_map); + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ds_ctx, &ns_info->ns_map); if (NULL == elem) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; @@ -1591,8 +961,8 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) /* directly compute index of meta segment (id) and relative offset (rel_offset) * inside this segment for fast lookup a rank_meta_info object for the requested rank. */ size_t rcount = rinfo->rank == PMIX_RANK_WILDCARD ? 0 : rinfo->rank + 1; - id = rcount/_max_meta_elems; - rel_offset = (rcount % _max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); + id = rcount/ds_ctx->max_meta_elems; + rel_offset = (rcount % ds_ctx->max_meta_elems) * sizeof(rank_meta_info) + sizeof(size_t); count = id; /* go through all existing meta segments for this namespace. * Stop at id number if it exists. */ @@ -1604,7 +974,8 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) if ((int)ns_info->num_meta_seg < (id+1)) { while ((int)ns_info->num_meta_seg != (id+1)) { /* extend meta segment, so create a new one */ - tmp = extend_segment(tmp, &ns_info->ns_map); + tmp = pmix_common_dstor_extend_segment(tmp, ds_ctx->base_path, ns_info->ns_map.name, + ds_ctx->jobuid, ds_ctx->setjobuid); if (NULL == tmp) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; @@ -1613,7 +984,7 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) ns_info->num_meta_seg++; } /* update number of meta segments for namespace in initial_segment */ - ns_seg_info_t *elem = _get_ns_info_from_initial_segment(&ns_info->ns_map); + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ds_ctx, &ns_info->ns_map); if (NULL == elem) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; @@ -1629,9 +1000,9 @@ static int set_rank_meta_info(ns_track_elem_t *ns_info, rank_meta_info *rinfo) return PMIX_SUCCESS; } -static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset) +static uint8_t *_get_data_region_by_offset(pmix_common_dstore_ctx_t *ds_ctx, pmix_dstore_seg_desc_t *segdesc, size_t offset) { - seg_desc_t *tmp = segdesc; + pmix_dstore_seg_desc_t *tmp = segdesc; size_t rel_offset = offset; uint8_t *dataaddr = NULL; @@ -1641,8 +1012,8 @@ static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset) /* go through all existing data segments for this namespace */ do { - if (rel_offset >= _data_segment_size) { - rel_offset -= _data_segment_size; + if (rel_offset >= ds_ctx->data_segment_size) { + rel_offset -= ds_ctx->data_segment_size; } else { dataaddr = tmp->seg_info.seg_base_addr + rel_offset; } @@ -1652,10 +1023,10 @@ static uint8_t *_get_data_region_by_offset(seg_desc_t *segdesc, size_t offset) return dataaddr; } -static size_t get_free_offset(seg_desc_t *data_seg) +static size_t get_free_offset(pmix_common_dstore_ctx_t *ds_ctx, pmix_dstore_seg_desc_t *data_seg) { size_t offset; - seg_desc_t *tmp; + pmix_dstore_seg_desc_t *tmp; int id = 0; tmp = data_seg; /* first find the last data segment */ @@ -1668,35 +1039,42 @@ static size_t get_free_offset(seg_desc_t *data_seg) /* this is the first created data segment, the first 8 bytes are used to place the free offset value itself */ offset = sizeof(size_t); } - return (id * _data_segment_size + offset); + return (id * ds_ctx->data_segment_size + offset); } -static int put_empty_ext_slot(seg_desc_t *dataseg) +static int put_empty_ext_slot(pmix_common_dstore_ctx_t *ds_ctx, pmix_dstore_seg_desc_t *dataseg) { size_t global_offset, rel_offset, data_ended, val = 0; uint8_t *addr; - global_offset = get_free_offset(dataseg); - rel_offset = global_offset % _data_segment_size; - if (rel_offset + EXT_SLOT_SIZE() > _data_segment_size) { + pmix_status_t rc; + + global_offset = get_free_offset(ds_ctx, dataseg); + rel_offset = global_offset % ds_ctx->data_segment_size; + if (rel_offset + PMIX_DS_SLOT_SIZE(ds_ctx) > ds_ctx->data_segment_size) { PMIX_ERROR_LOG(PMIX_ERROR); return PMIX_ERROR; } - addr = _get_data_region_by_offset(dataseg, global_offset); - ESH_PUT_KEY(addr, ESH_REGION_EXTENSION, (void*)&val, sizeof(size_t)); - + addr = _get_data_region_by_offset(ds_ctx, dataseg, global_offset); + PMIX_DS_PUT_KEY(rc, ds_ctx, addr, ESH_REGION_EXTENSION, (void*)&val, sizeof(size_t)); + if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); + return rc; + } /* update offset at the beginning of current segment */ - data_ended = rel_offset + EXT_SLOT_SIZE(); + data_ended = rel_offset + PMIX_DS_SLOT_SIZE(ds_ctx); addr = (uint8_t*)(addr - rel_offset); memcpy(addr, &data_ended, sizeof(size_t)); return PMIX_SUCCESS; } -static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, char *key, void *buffer, size_t size) +static size_t put_data_to_the_end(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, + pmix_dstore_seg_desc_t *dataseg, char *key, void *buffer, size_t size) { size_t offset, id = 0; - seg_desc_t *tmp; + pmix_dstore_seg_desc_t *tmp; size_t global_offset, data_ended; uint8_t *addr; + pmix_status_t rc; PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, "%s:%d:%s: key %s", @@ -1707,17 +1085,19 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, tmp = tmp->next; id++; } - global_offset = get_free_offset(dataseg); - offset = global_offset % _data_segment_size; + global_offset = get_free_offset(ds_ctx, dataseg); + offset = global_offset % ds_ctx->data_segment_size; /* We should provide additional space at the end of segment to * place EXTENSION_SLOT to have an ability to enlarge data for this rank.*/ - if ((sizeof(size_t) + ESH_KEY_SIZE(key, size) + EXT_SLOT_SIZE()) > _data_segment_size) { + if ((sizeof(size_t) + PMIX_DS_KEY_SIZE(ds_ctx, key, size) + PMIX_DS_SLOT_SIZE(ds_ctx)) > + ds_ctx->data_segment_size) { /* this is an error case: segment is so small that cannot place evem a single key-value pair. * warn a user about it and fail. */ offset = 0; /* offset cannot be 0 in normal case, so we use this value to indicate a problem. */ pmix_output(0, "PLEASE set NS_DATA_SEG_SIZE to value which is larger when %lu.", - (unsigned long)(sizeof(size_t) + strlen(key) + 1 + sizeof(size_t) + size + EXT_SLOT_SIZE())); + (unsigned long)(sizeof(size_t) + strlen(key) + 1 + sizeof(size_t) + + size + PMIX_DS_SLOT_SIZE(ds_ctx))); return offset; } @@ -1728,10 +1108,12 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, * new segment wasn't allocated to us but (global_offset % _data_segment_size) == 0 * so if offset is 0 here - we need to allocate the segment as well */ - if ( (0 == offset) || ( (offset + ESH_KEY_SIZE(key, size) + EXT_SLOT_SIZE()) > _data_segment_size) ) { + if ( (0 == offset) || ( (offset + PMIX_DS_KEY_SIZE(ds_ctx, key, size) + + PMIX_DS_SLOT_SIZE(ds_ctx)) > ds_ctx->data_segment_size) ) { id++; /* create a new data segment. */ - tmp = extend_segment(tmp, &ns_info->ns_map); + tmp = pmix_common_dstor_extend_segment(tmp, ds_ctx->base_path, ns_info->ns_map.name, + ds_ctx->jobuid, ds_ctx->setjobuid); if (NULL == tmp) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); offset = 0; /* offset cannot be 0 in normal case, so we use this value to indicate a problem. */ @@ -1739,7 +1121,7 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, } ns_info->num_data_seg++; /* update_ns_info_in_initial_segment */ - ns_seg_info_t *elem = _get_ns_info_from_initial_segment(&ns_info->ns_map); + ns_seg_info_t *elem = _get_ns_info_from_initial_segment(ds_ctx, &ns_info->ns_map); if (NULL == elem) { PMIX_ERROR_LOG(PMIX_ERR_NOMEM); offset = 0; /* offset cannot be 0 in normal case, so we use this value to indicate a problem. */ @@ -1748,12 +1130,16 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, elem->num_data_seg++; offset = sizeof(size_t); } - global_offset = offset + id * _data_segment_size; + global_offset = offset + id * ds_ctx->data_segment_size; addr = (uint8_t*)(tmp->seg_info.seg_base_addr)+offset; - ESH_PUT_KEY(addr, key, buffer, size); + PMIX_DS_PUT_KEY(rc, ds_ctx, addr, key, buffer, size); + if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); + return 0; + } /* update offset at the beginning of current segment */ - data_ended = offset + ESH_KEY_SIZE(key, size); + data_ended = offset + PMIX_DS_KEY_SIZE(ds_ctx, key, size); addr = (uint8_t*)(tmp->seg_info.seg_base_addr); memcpy(addr, &data_ended, sizeof(size_t)); PMIX_OUTPUT_VERBOSE((1, pmix_gds_base_framework.framework_output, @@ -1761,17 +1147,18 @@ static size_t put_data_to_the_end(ns_track_elem_t *ns_info, seg_desc_t *dataseg, __FILE__, __LINE__, __func__, key, (unsigned long)offset, (unsigned long)data_ended, - (unsigned long)(id * _data_segment_size), + (unsigned long)(id * ds_ctx->data_segment_size), (unsigned long)size)); return global_offset; } -static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t *kval, rank_meta_info **rinfo, int data_exist) +static int pmix_sm_store(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, + pmix_rank_t rank, pmix_kval_t *kval, rank_meta_info **rinfo, int data_exist) { size_t offset, size, kval_cnt; pmix_buffer_t buffer; pmix_status_t rc; - seg_desc_t *datadesc; + pmix_dstore_seg_desc_t *datadesc; uint8_t *addr; PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, @@ -1781,7 +1168,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t datadesc = ns_info->data_seg; /* pack value to the buffer */ PMIX_CONSTRUCT(&buffer, pmix_buffer_t); - PMIX_BFROPS_PACK(rc, _client_peer(), &buffer, kval->value, 1, PMIX_VALUE); + PMIX_BFROPS_PACK(rc, _client_peer(ds_ctx), &buffer, kval->value, 1, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto exit; @@ -1791,8 +1178,8 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t if (0 == data_exist) { /* there is no data blob for this rank yet, so add it. */ size_t free_offset; - free_offset = get_free_offset(datadesc); - offset = put_data_to_the_end(ns_info, datadesc, kval->key, buffer.base_ptr, size); + free_offset = get_free_offset(ds_ctx, datadesc); + offset = put_data_to_the_end(ds_ctx, ns_info, datadesc, kval->key, buffer.base_ptr, size); if (0 == offset) { /* this is an error */ rc = PMIX_ERROR; @@ -1806,11 +1193,15 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * because previous segment is already full. */ if (free_offset != offset && NULL != *rinfo) { /* here we compare previous free offset with the offset where we just put data. - * It should be equal in the normal case. It it's not true, then it means that + * It should be equal in the normal case. If it's not true, then it means that * segment was extended, and we put data to the next segment, so we now need to * put extension slot at the end of previous segment with a "reference" to a new_offset */ - addr = _get_data_region_by_offset(datadesc, free_offset); - ESH_PUT_KEY(addr, ESH_REGION_EXTENSION, (void*)&offset, sizeof(size_t)); + addr = _get_data_region_by_offset(ds_ctx, datadesc, free_offset); + PMIX_DS_PUT_KEY(rc, ds_ctx, addr, ESH_REGION_EXTENSION, (void*)&offset, sizeof(size_t)); + if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); + return 0; + } } if (NULL == *rinfo) { *rinfo = (rank_meta_info*)malloc(sizeof(rank_meta_info)); @@ -1821,7 +1212,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t (*rinfo)->count++; } else if (NULL != *rinfo) { /* there is data blob for this rank */ - addr = _get_data_region_by_offset(datadesc, (*rinfo)->offset); + addr = _get_data_region_by_offset(ds_ctx, datadesc, (*rinfo)->offset); if (NULL == addr) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); @@ -1843,8 +1234,8 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * ..... * extension slot which has key = EXTENSION_SLOT and a size_t value for offset to next data address for this process. */ - if (0 == strncmp(ESH_KNAME_PTR(addr), ESH_REGION_EXTENSION, ESH_KNAME_LEN(ESH_REGION_EXTENSION))) { - memcpy(&offset, ESH_DATA_PTR(addr), sizeof(size_t)); + if(PMIX_DS_KEY_IS_EXTSLOT(ds_ctx, addr)) { + memcpy(&offset, PMIX_DS_DATA_PTR(ds_ctx, addr), sizeof(size_t)); if (0 < offset) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %lu, replace flag %d %s is filled with %lu value", @@ -1852,7 +1243,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t (unsigned long)rank, data_exist, ESH_REGION_EXTENSION, (unsigned long)offset)); /* go to next item, updating address */ - addr = _get_data_region_by_offset(datadesc, offset); + addr = _get_data_region_by_offset(ds_ctx, datadesc, offset); if (NULL == addr) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); @@ -1861,20 +1252,21 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t } else { /* should not be, we should be out of cycle when this happens */ } - } else if (0 == strncmp(ESH_KNAME_PTR(addr), kval->key, ESH_KNAME_LEN(kval->key))) { + } else if (0 == strncmp(PMIX_DS_KNAME_PTR(ds_ctx, addr), kval->key, + PMIX_DS_KNAME_LEN(ds_ctx, kval->key))) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %u, replace flag %d found target key %s", __FILE__, __LINE__, __func__, rank, data_exist, kval->key)); /* target key is found, compare value sizes */ - if (ESH_DATA_SIZE(addr, ESH_DATA_PTR(addr)) != size) { + if (PMIX_DS_DATA_SIZE(ds_ctx, addr, PMIX_DS_DATA_PTR(ds_ctx, addr)) != size) { //if (1) { /* if we want to test replacing values for existing keys. */ /* invalidate current value and store another one at the end of data region. */ - strncpy(ESH_KNAME_PTR(addr), ESH_REGION_INVALIDATED, ESH_KNAME_LEN(ESH_REGION_INVALIDATED)); + PMIX_DS_KEY_SET_INVALID(ds_ctx, addr); /* decrementing count, it will be incremented back when we add a new value for this key at the end of region. */ (*rinfo)->count--; kval_cnt--; /* go to next item, updating address */ - addr += ESH_KV_SIZE(addr); + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %u, replace flag %d mark key %s regions as invalidated. put new data at the end.", __FILE__, __LINE__, __func__, rank, data_exist, kval->key)); @@ -1883,23 +1275,25 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t "%s:%d:%s: for rank %u, replace flag %d replace data for key %s type %d in place", __FILE__, __LINE__, __func__, rank, data_exist, kval->key, kval->value->type)); /* replace old data with new one. */ - memset(ESH_DATA_PTR(addr), 0, ESH_DATA_SIZE(addr, ESH_DATA_PTR(addr))); - memcpy(ESH_DATA_PTR(addr), buffer.base_ptr, size); - addr += ESH_KV_SIZE(addr); + memset(PMIX_DS_DATA_PTR(ds_ctx, addr), 0, + PMIX_DS_DATA_SIZE(ds_ctx, addr, PMIX_DS_DATA_PTR(ds_ctx, addr))); + memcpy(PMIX_DS_DATA_PTR(ds_ctx, addr), buffer.base_ptr, size); + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); add_to_the_end = 0; break; } } else { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %u, replace flag %d skip %s key, look for %s key", - __FILE__, __LINE__, __func__, rank, data_exist, ESH_KNAME_PTR(addr), kval->key)); + __FILE__, __LINE__, __func__, rank, data_exist, + PMIX_DS_KNAME_PTR(ds_ctx, addr), kval->key)); /* Skip it: key is "INVALIDATED" or key is valid but different from target one. */ - if (0 != strncmp(ESH_REGION_INVALIDATED, ESH_KNAME_PTR(addr), ESH_KNAME_LEN(ESH_KNAME_PTR(addr)))) { + if (!PMIX_DS_KEY_IS_INVALID(ds_ctx, addr)) { /* count only valid items */ kval_cnt--; } /* go to next item, updating address */ - addr += ESH_KV_SIZE(addr); + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); } } if (1 == add_to_the_end) { @@ -1908,9 +1302,68 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * for the same key. */ size_t free_offset; (*rinfo)->count++; - free_offset = get_free_offset(datadesc); + free_offset = get_free_offset(ds_ctx, datadesc); + + /* + * Remove trailing extention slot if we are continuing + * same ranks data. + * + * When keys are stored individually through _store_data_for_rank + * an empty extention slot is placed every time. + * + * This is required because there is no information about whether or not the next key + * will belong to the same rank. + * + * As the result EACH keys stored with _store_data_for_rank is + * followed by extension slot. This slows down search and increases + * the memory footprint. + * + * The following code tries to deal with such one-key-at-a-time + * situation by: + * - checking if the last key-value for this rank is an extention + * slot + * - If this is the case - checks if this key-value pair is the + * last one at the moment and can be safely deleted. + * - if it is - current segment's offset pointer is decreased by + * the size of the extention slot key-value effectively removing + * it from the dstor + */ + if (PMIX_DS_KEY_IS_EXTSLOT(ds_ctx, addr)){ + /* Find the last data segment */ + pmix_dstore_seg_desc_t *ldesc = datadesc; + uint8_t *segstart; + size_t offs_past_extslot = 0; + size_t offs_cur_segment = 0; + while (NULL != ldesc->next) { + ldesc = ldesc->next; + } + + /* Calculate the offset of the end of the extension slot */ + offs_cur_segment = free_offset % ds_ctx->data_segment_size; + segstart = ldesc->seg_info.seg_base_addr; + offs_past_extslot = (addr + PMIX_DS_KV_SIZE(ds_ctx, addr)) - segstart; + + /* We can erase extension slot if: + * - address of the ext slot belongs to the occupied part of the + * last segment + * - local offset within the segment is equal to the local + * offset of the end of extension slot + */ + if( ( (addr > segstart) && (addr < (segstart + offs_cur_segment)) ) + && (offs_cur_segment == offs_past_extslot) ) { + /* Calculate a new free offset that doesn't account this + * extension slot */ + size_t new_offset = addr - segstart; + /* Rewrite segment's offset information to exclude + * extension slot */ + memcpy(segstart, &new_offset, sizeof(size_t)); + /* Recalculate free_offset */ + free_offset = get_free_offset(ds_ctx, datadesc); + } + } + /* add to the end */ - offset = put_data_to_the_end(ns_info, datadesc, kval->key, buffer.base_ptr, size); + offset = put_data_to_the_end(ds_ctx, ns_info, datadesc, kval->key, buffer.base_ptr, size); if (0 == offset) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); @@ -1921,11 +1374,11 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * data for different ranks, and that's why next element is EXTENSION_SLOT. * We put new data to the end of data region and just update EXTENSION_SLOT value by new offset. */ - if (0 == strncmp(ESH_KNAME_PTR(addr), ESH_REGION_EXTENSION, ESH_KNAME_LEN(ESH_REGION_EXTENSION))) { + if (PMIX_DS_KEY_IS_EXTSLOT(ds_ctx, addr)) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %u, replace flag %d %s should be filled with offset %lu value", __FILE__, __LINE__, __func__, rank, data_exist, ESH_REGION_EXTENSION, offset)); - memcpy(ESH_DATA_PTR(addr), &offset, sizeof(size_t)); + memcpy(PMIX_DS_DATA_PTR(ds_ctx, addr), &offset, sizeof(size_t)); } else { /* (2) - we point to the first free offset, no more data is stored further in this segment. * There is no EXTENSION_SLOT by this addr since we continue pushing data for the same rank, @@ -1935,7 +1388,11 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t * forcibly and store new offset in its value. */ if (free_offset != offset) { /* segment was extended, need to put extension slot by free_offset indicating new_offset */ - ESH_PUT_KEY(addr, ESH_REGION_EXTENSION, (void*)&offset, sizeof(size_t)); + PMIX_DS_PUT_KEY(rc, ds_ctx, addr, ESH_REGION_EXTENSION, (void*)&offset, sizeof(size_t)); + if (rc != PMIX_SUCCESS) { + PMIX_ERROR_LOG(rc); + return 0; + } } } PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, @@ -1948,12 +1405,13 @@ exit: return rc; } -static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_buffer_t *buf) +static int _store_data_for_rank(pmix_common_dstore_ctx_t *ds_ctx, ns_track_elem_t *ns_info, + pmix_rank_t rank, pmix_buffer_t *buf) { pmix_status_t rc; pmix_kval_t *kp; - seg_desc_t *metadesc, *datadesc; + pmix_dstore_seg_desc_t *metadesc, *datadesc; int32_t cnt; rank_meta_info *rinfo = NULL; @@ -1974,11 +1432,11 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix num_elems = *((size_t*)(metadesc->seg_info.seg_base_addr)); data_exist = 0; - /* when we don't use linear search (_direct_mode ==0 ) we don't use num_elems field, + /* when we don't use linear search (direct_mode == 0) we don't use num_elems field, * so anyway try to get rank_meta_info first. */ - if (0 < num_elems || 0 == _direct_mode) { + if (0 < num_elems || 0 == ds_ctx->direct_mode) { /* go through all elements in meta segment and look for target rank. */ - rinfo = _get_rank_meta_info(rank, metadesc); + rinfo = _get_rank_meta_info(ds_ctx, rank, metadesc); if (NULL != rinfo) { data_exist = 1; } @@ -1987,14 +1445,14 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix * so unpack these buffers, and then unpack kvals from each modex buffer, * storing them in the shared memory dstore. */ - free_offset = get_free_offset(datadesc); + free_offset = get_free_offset(ds_ctx, datadesc); cnt = 1; kp = PMIX_NEW(pmix_kval_t); PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, buf, kp, &cnt, PMIX_KVAL); while(PMIX_SUCCESS == rc) { pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "pmix: unpacked key %s", kp->key); - if (PMIX_SUCCESS != (rc = pmix_sm_store(ns_info, rank, kp, &rinfo, data_exist))) { + if (PMIX_SUCCESS != (rc = pmix_sm_store(ds_ctx, ns_info, rank, kp, &rinfo, data_exist))) { PMIX_ERROR_LOG(rc); if (NULL != rinfo) { free(rinfo); @@ -2021,7 +1479,7 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix * in that case we don't reserve space for EXTENSION_SLOT, it's * already reserved. * */ - new_free_offset = get_free_offset(datadesc); + new_free_offset = get_free_offset(ds_ctx, datadesc); if (new_free_offset != free_offset) { /* Reserve space for EXTENSION_SLOT at the end of data blob. * We need it to split data for one rank from data for different @@ -2029,7 +1487,7 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix * We also put EXTENSION_SLOT at the end of each data segment, and * its value points to the beginning of next data segment. * */ - rc = put_empty_ext_slot(ns_info->data_seg); + rc = put_empty_ext_slot(ds_ctx, ns_info->data_seg); if (PMIX_SUCCESS != rc) { if ((0 == data_exist) && NULL != rinfo) { free(rinfo); @@ -2042,7 +1500,7 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix /* if this is the first data posted for this rank, then * update meta info for it */ if (0 == data_exist) { - set_rank_meta_info(ns_info, rinfo); + set_rank_meta_info(ds_ctx, ns_info, rinfo); if (NULL != rinfo) { free(rinfo); } @@ -2051,13 +1509,13 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix return rc; } -static inline ssize_t _get_univ_size(const char *nspace) +static inline ssize_t _get_univ_size(pmix_common_dstore_ctx_t *ds_ctx, const char *nspace) { ssize_t nprocs = 0; pmix_value_t *val; int rc; - rc = _dstore_fetch(nspace, PMIX_RANK_WILDCARD, PMIX_UNIV_SIZE, &val); + rc = _dstore_fetch(ds_ctx, nspace, PMIX_RANK_WILDCARD, PMIX_UNIV_SIZE, &val); if( PMIX_SUCCESS != rc ) { PMIX_ERROR_LOG(rc); return rc; @@ -2072,23 +1530,39 @@ static inline ssize_t _get_univ_size(const char *nspace) return nprocs; } -static pmix_status_t dstore_cache_job_info(struct pmix_nspace_t *ns, +PMIX_EXPORT pmix_status_t pmix_common_dstor_cache_job_info(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *ns, pmix_info_t info[], size_t ninfo) { return PMIX_SUCCESS; } -static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo) + +pmix_common_dstore_ctx_t *pmix_common_dstor_init(const char *ds_name, pmix_info_t info[], size_t ninfo, + pmix_common_lock_callbacks_t *lock_cb, + pmix_common_dstore_file_cbs_t *file_cb) { pmix_status_t rc; size_t n; char *dstor_tmpdir = NULL; - size_t tbl_idx=0; + size_t tbl_idx = 0; ns_map_data_t *ns_map = NULL; + pmix_common_dstore_ctx_t *ds_ctx = NULL; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "pmix:gds:dstore init"); + ds_ctx = (pmix_common_dstore_ctx_t*) malloc(sizeof(*ds_ctx)); + if (NULL == ds_ctx) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + return NULL; + } + memset(ds_ctx, 0, sizeof(*ds_ctx)); + + /* assign lock callbacks */ + ds_ctx->lock_cbs = lock_cb; + ds_ctx->file_cbs = file_cb; + /* open the pshmem and select the active plugins */ if( PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pshmem_base_framework, 0)) ) { PMIX_ERROR_LOG(rc); @@ -2099,17 +1573,10 @@ static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo) goto err_exit; } - _jobuid = getuid(); - _setjobuid = 0; + ds_ctx->jobuid = getuid(); + ds_ctx->setjobuid = 0; -#ifdef ESH_PTHREAD_LOCK - _esh_lock_init = _rwlock_init; -#endif -#ifdef ESH_FCNTL_LOCK - _esh_lock_init = _flock_init; -#endif - - if (PMIX_SUCCESS != (rc = _esh_tbls_init())) { + if (PMIX_SUCCESS != (rc = _esh_tbls_init(ds_ctx))) { PMIX_ERROR_LOG(rc); goto err_exit; } @@ -2120,23 +1587,19 @@ static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo) goto err_exit; } - _set_constants_from_env(); - - if (NULL != _base_path) { - free(_base_path); - _base_path = NULL; - } + _set_constants_from_env(ds_ctx); + ds_ctx->ds_name = strdup(ds_name); /* find the temp dir */ if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - _esh_session_map_search = _esh_session_map_search_server; + ds_ctx->session_map_search = (session_map_search_fn_t)_esh_session_map_search_server; /* scan incoming info for directives */ if (NULL != info) { for (n=0; n < ninfo; n++) { if (0 == strcmp(PMIX_USERID, info[n].key)) { - _jobuid = info[n].value.data.uint32; - _setjobuid = 1; + ds_ctx->jobuid = info[n].value.data.uint32; + ds_ctx->setjobuid = 1; continue; } if (0 == strcmp(PMIX_DSTPATH, info[n].key)) { @@ -2182,67 +1645,109 @@ static pmix_status_t dstore_init(pmix_info_t info[], size_t ninfo) } } - rc = asprintf(&_base_path, "%s/pmix_dstor_%d", dstor_tmpdir, getpid()); - if ((0 > rc) || (NULL == _base_path)) { + rc = asprintf(&ds_ctx->base_path, "%s/pmix_dstor_%s_%d", dstor_tmpdir, + ds_ctx->ds_name, getpid()); + if ((0 > rc) || (NULL == ds_ctx->base_path)) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (0 != mkdir(_base_path, 0770)) { + if (0 != mkdir(ds_ctx->base_path, 0770)) { if (EEXIST != errno) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); goto err_exit; } } - if (_setjobuid > 0) { - if (chown(_base_path, (uid_t) _jobuid, (gid_t) -1) < 0){ + if (ds_ctx->setjobuid > 0) { + if (chown(ds_ctx->base_path, (uid_t) ds_ctx->jobuid, (gid_t) -1) < 0){ rc = PMIX_ERR_NO_PERMISSIONS; PMIX_ERROR_LOG(rc); goto err_exit; } } - _esh_session_map_search = _esh_session_map_search_server; - return PMIX_SUCCESS; + ds_ctx->session_map_search = _esh_session_map_search_server; + return ds_ctx; } /* for clients */ else { - if (NULL == (dstor_tmpdir = getenv(PMIX_DSTORE_ESH_BASE_PATH))){ - return PMIX_ERR_NOT_AVAILABLE; // simply disqualify ourselves + char *env_name = NULL; + int ds_ver = 0; + + sscanf(ds_ctx->ds_name, "ds%d", &ds_ver); + if (0 == ds_ver) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto err_exit; } - if (NULL == (_base_path = strdup(dstor_tmpdir))) { + if (0 > asprintf(&env_name, PMIX_DSTORE_VER_BASE_PATH_FMT, ds_ver)) { + rc = PMIX_ERR_NOMEM; + PMIX_ERROR_LOG(rc); + goto err_exit; + } + dstor_tmpdir = getenv(env_name); + free(env_name); + + if (NULL == dstor_tmpdir) { + dstor_tmpdir = getenv(PMIX_DSTORE_ESH_BASE_PATH); + } + if (NULL == dstor_tmpdir){ + rc = PMIX_ERR_NOT_AVAILABLE; // simply disqualify ourselves + goto err_exit; + } + if (NULL == (ds_ctx->base_path = strdup(dstor_tmpdir))) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - _esh_session_map_search = _esh_session_map_search_client; + ds_ctx->session_map_search = _esh_session_map_search_client; + /* init ds_ctx protect lock */ + if (0 != pthread_mutex_init(&ds_ctx->lock, NULL)) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto err_exit; + } } - rc = _esh_session_tbl_add(&tbl_idx); + rc = _esh_session_tbl_add(ds_ctx, &tbl_idx); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto err_exit; } - ns_map = _esh_session_map(pmix_globals.myid.nspace, tbl_idx); + char *nspace = NULL; + /* if we don't see the required info, then we cannot init */ + if (NULL == (nspace = getenv("PMIX_NAMESPACE"))) { + rc = PMIX_ERR_INVALID_NAMESPACE; + PMIX_ERROR_LOG(rc); + goto err_exit; + } + /* lock init */ + rc = ds_ctx->lock_cbs->init(&_ESH_SESSION_lock(ds_ctx->session_array, tbl_idx), ds_ctx->base_path, nspace, 1, ds_ctx->jobuid, ds_ctx->setjobuid); + if (rc != PMIX_SUCCESS) { + goto err_exit; + } + ns_map = _esh_session_map(ds_ctx, nspace, 0, tbl_idx); if (NULL == ns_map) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); goto err_exit; } - if (PMIX_SUCCESS != (rc =_esh_session_init(tbl_idx, ns_map, _jobuid, _setjobuid))) { + if (PMIX_SUCCESS != (rc =_esh_session_init(ds_ctx, tbl_idx, ns_map, 1, + ds_ctx->jobuid, ds_ctx->setjobuid))) { PMIX_ERROR_LOG(rc); goto err_exit; } - return PMIX_SUCCESS; + return ds_ctx; err_exit: - return rc; + pmix_common_dstor_finalize(ds_ctx); + return NULL; } -static void dstore_finalize(void) +PMIX_EXPORT void pmix_common_dstor_finalize(pmix_common_dstore_ctx_t *ds_ctx) { struct stat st = {0}; pmix_status_t rc = PMIX_SUCCESS; @@ -2250,42 +1755,45 @@ static void dstore_finalize(void) PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s", __FILE__, __LINE__, __func__)); - _esh_sessions_cleanup(); - _esh_ns_map_cleanup(); - _esh_ns_track_cleanup(); + _esh_sessions_cleanup(ds_ctx); + _esh_ns_map_cleanup(ds_ctx); + _esh_ns_track_cleanup(ds_ctx); pmix_pshmem.finalize(); - if (NULL != _base_path){ + if (NULL != ds_ctx->base_path){ if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { - if (lstat(_base_path, &st) >= 0){ - if (PMIX_SUCCESS != (rc = _esh_dir_del(_base_path))) { + if (lstat(ds_ctx->base_path, &st) >= 0){ + if (PMIX_SUCCESS != (rc = _esh_dir_del(ds_ctx->base_path))) { PMIX_ERROR_LOG(rc); } } } - free(_base_path); - _base_path = NULL; + free(ds_ctx->base_path); + ds_ctx->base_path = NULL; } - if (NULL != _clients_peer) { - PMIX_RELEASE(_clients_peer->nptr); - PMIX_RELEASE(_clients_peer); + if (NULL != ds_ctx->clients_peer) { + PMIX_RELEASE(ds_ctx->clients_peer->nptr); + PMIX_RELEASE(ds_ctx->clients_peer); } /* close the pshmem framework */ if( PMIX_SUCCESS != (rc = pmix_mca_base_framework_close(&pmix_pshmem_base_framework)) ) { PMIX_ERROR_LOG(rc); } + free(ds_ctx->ds_name); + free(ds_ctx->base_path); + free(ds_ctx); } -static pmix_status_t _dstore_store(const char *nspace, - pmix_rank_t rank, - pmix_kval_t *kv) +static pmix_status_t _dstore_store_nolock(pmix_common_dstore_ctx_t *ds_ctx, + ns_map_data_t *ns_map, + pmix_rank_t rank, + pmix_kval_t *kv) { - pmix_status_t rc = PMIX_SUCCESS, tmp_rc; + pmix_status_t rc = PMIX_SUCCESS; ns_track_elem_t *elem; pmix_buffer_t xfer; ns_seg_info_t ns_info; - ns_map_data_t *ns_map = NULL; if (NULL == kv) { return PMIX_ERROR; @@ -2293,19 +1801,7 @@ static pmix_status_t _dstore_store(const char *nspace, PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for %s:%u", - __FILE__, __LINE__, __func__, nspace, rank)); - - if (NULL == (ns_map = _esh_session_map_search(nspace))) { - rc = PMIX_ERROR; - PMIX_ERROR_LOG(rc); - return rc; - } - - /* set exclusive lock */ - if (PMIX_SUCCESS != (rc = _ESH_WRLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { - PMIX_ERROR_LOG(rc); - return rc; - } + __FILE__, __LINE__, __func__, ns_map->name, rank)); /* First of all, we go through local track list (list of ns_track_elem_t structures) * and look for an element for the target namespace. @@ -2317,36 +1813,36 @@ static pmix_status_t _dstore_store(const char *nspace, * All this stuff is done inside _get_track_elem_for_namespace function. */ - elem = _get_track_elem_for_namespace(ns_map); + elem = _get_track_elem_for_namespace(ds_ctx, ns_map); if (NULL == elem) { rc = PMIX_ERR_OUT_OF_RESOURCE; PMIX_ERROR_LOG(rc); - goto err_exit; + goto exit; } /* If a new element was just created, we need to create corresponding meta and * data segments and update corresponding element's fields. */ if (NULL == elem->meta_seg || NULL == elem->data_seg) { memset(&ns_info.ns_map, 0, sizeof(ns_info.ns_map)); - strncpy(ns_info.ns_map.name, ns_map->name, sizeof(ns_info.ns_map.name)-1); + pmix_strncpy(ns_info.ns_map.name, ns_map->name, sizeof(ns_info.ns_map.name)-1); ns_info.ns_map.tbl_idx = ns_map->tbl_idx; ns_info.num_meta_seg = 1; ns_info.num_data_seg = 1; - rc = _update_ns_elem(elem, &ns_info); + rc = _update_ns_elem(ds_ctx, elem, &ns_info); if (PMIX_SUCCESS != rc || NULL == elem->meta_seg || NULL == elem->data_seg) { PMIX_ERROR_LOG(rc); - goto err_exit; + goto exit; } /* zero created shared memory segments for this namespace */ - memset(elem->meta_seg->seg_info.seg_base_addr, 0, _meta_segment_size); - memset(elem->data_seg->seg_info.seg_base_addr, 0, _data_segment_size); + memset(elem->meta_seg->seg_info.seg_base_addr, 0, ds_ctx->meta_segment_size); + memset(elem->data_seg->seg_info.seg_base_addr, 0, ds_ctx->data_segment_size); /* put ns's shared segments info to the global meta segment. */ - rc = _put_ns_info_to_initial_segment(ns_map, &elem->meta_seg->seg_info, &elem->data_seg->seg_info); + rc = _put_ns_info_to_initial_segment(ds_ctx, ns_map, &elem->meta_seg->seg_info, &elem->data_seg->seg_info); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto err_exit; + goto exit; } } @@ -2355,34 +1851,28 @@ static pmix_status_t _dstore_store(const char *nspace, PMIX_CONSTRUCT(&xfer, pmix_buffer_t); PMIX_LOAD_BUFFER(pmix_globals.mypeer, &xfer, kv->value->data.bo.bytes, kv->value->data.bo.size); - rc = _store_data_for_rank(elem, rank, &xfer); + rc = _store_data_for_rank(ds_ctx, elem, rank, &xfer); PMIX_DESTRUCT(&xfer); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto err_exit; + goto exit; } - /* unset lock */ - if (PMIX_SUCCESS != (rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { - PMIX_ERROR_LOG(rc); - } - return rc; - -err_exit: - /* unset lock */ - if (PMIX_SUCCESS != (tmp_rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { - PMIX_ERROR_LOG(tmp_rc); - } +exit: return rc; } -static pmix_status_t dstore_store(const pmix_proc_t *proc, - pmix_scope_t scope, - pmix_kval_t *kv) +PMIX_EXPORT pmix_status_t pmix_common_dstor_store(pmix_common_dstore_ctx_t *ds_ctx, + const pmix_proc_t *proc, + pmix_scope_t scope, + pmix_kval_t *kv) { pmix_status_t rc = PMIX_SUCCESS; + ns_map_data_t *ns_map; + pmix_kval_t *kv2; + pmix_buffer_t tmp; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "[%s:%d] gds: dstore store for key '%s' scope %d", @@ -2393,26 +1883,51 @@ static pmix_status_t dstore_store(const pmix_proc_t *proc, PMIX_ERROR_LOG(rc); return rc; } - else { - pmix_kval_t *kv2; - kv2 = PMIX_NEW(pmix_kval_t); - PMIX_VALUE_CREATE(kv2->value, 1); - kv2->value->type = PMIX_BYTE_OBJECT; - pmix_buffer_t tmp; - PMIX_CONSTRUCT(&tmp, pmix_buffer_t); + kv2 = PMIX_NEW(pmix_kval_t); + PMIX_VALUE_CREATE(kv2->value, 1); + kv2->value->type = PMIX_BYTE_OBJECT; - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &tmp, kv, 1, PMIX_KVAL); - PMIX_UNLOAD_BUFFER(&tmp, kv2->value->data.bo.bytes, kv2->value->data.bo.size); + PMIX_CONSTRUCT(&tmp, pmix_buffer_t); - rc = _dstore_store(proc->nspace, proc->rank, kv2); - PMIX_RELEASE(kv2); - PMIX_DESTRUCT(&tmp); + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &tmp, kv, 1, PMIX_KVAL); + PMIX_UNLOAD_BUFFER(&tmp, kv2->value->data.bo.bytes, kv2->value->data.bo.size); + + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, proc->nspace))) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto exit; } + + /* set exclusive lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_lock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto exit; + } + + rc = _dstore_store_nolock(ds_ctx, ns_map, proc->rank, kv2); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto exit; + } + + /* unset lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_unlock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto exit; + } + +exit: + PMIX_RELEASE(kv2); + PMIX_DESTRUCT(&tmp); + return rc; } -static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, +static pmix_status_t _dstore_fetch(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, pmix_rank_t rank, const char *key, pmix_value_t **kvs) { ns_seg_info_t *ns_info = NULL; @@ -2420,7 +1935,7 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, ns_track_elem_t *elem; rank_meta_info *rinfo = NULL; size_t kval_cnt = 0; - seg_desc_t *meta_seg, *data_seg; + pmix_dstore_seg_desc_t *meta_seg, *data_seg; uint8_t *addr; pmix_buffer_t buffer; pmix_value_t val, *kval = NULL; @@ -2431,6 +1946,8 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, bool key_found = false; pmix_info_t *info = NULL; size_t ninfo; + size_t keyhash = 0; + bool lock_is_set = false; PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for %s:%u look for key %s", @@ -2440,34 +1957,38 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, PMIX_OUTPUT_VERBOSE((7, pmix_gds_base_framework.framework_output, "dstore: Does not support passed parameters")); rc = PMIX_ERR_BAD_PARAM; - PMIX_ERROR_LOG(rc); - return rc; + goto error; } PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for %s:%u look for key %s", __FILE__, __LINE__, __func__, nspace, rank, key)); - if (NULL == (ns_map = _esh_session_map_search(nspace))) { + /* protect info of dstore segments before it will be updated */ + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (0 != (rc = pthread_mutex_lock(&ds_ctx->lock))) { + goto error; + } + lock_is_set = true; + } + + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, nspace))) { /* This call is issued from the the client. * client must have the session, otherwise the error is fatal. */ rc = PMIX_ERR_FATAL; - PMIX_ERROR_LOG(rc); - return rc; + goto error; } if (NULL == kvs) { rc = PMIX_ERR_FATAL; - PMIX_ERROR_LOG(rc); - return rc; + goto error; } if (PMIX_RANK_UNDEF == rank) { - ssize_t _nprocs = _get_univ_size(ns_map->name); + ssize_t _nprocs = _get_univ_size(ds_ctx, ns_map->name); if( 0 > _nprocs ){ - PMIX_ERROR_LOG(rc); - return rc; + goto error; } nprocs = (size_t) _nprocs; cur_rank = 0; @@ -2477,11 +1998,11 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } /* grab shared lock */ - if (PMIX_SUCCESS != (lock_rc = _ESH_RDLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { + lock_rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, rd_lock); + if (PMIX_SUCCESS != lock_rc) { /* Something wrong with the lock. The error is fatal */ - rc = PMIX_ERR_FATAL; - PMIX_ERROR_LOG(lock_rc); - return lock_rc; + rc = lock_rc; + goto error; } /* First of all, we go through all initial segments and look at their field. @@ -2497,9 +2018,9 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, */ /* first update local information about initial segments. they can be extended, so then we need to attach to new segments. */ - _update_initial_segment_info(ns_map); + _update_initial_segment_info(ds_ctx, ns_map); - ns_info = _get_ns_info_from_initial_segment(ns_map); + ns_info = _get_ns_info_from_initial_segment(ds_ctx, ns_map); if (NULL == ns_info) { /* no data for this namespace is found in the shared memory. */ PMIX_OUTPUT_VERBOSE((7, pmix_gds_base_framework.framework_output, @@ -2510,7 +2031,7 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } /* get ns_track_elem_t object for the target namespace from the local track list. */ - elem = _get_track_elem_for_namespace(ns_map); + elem = _get_track_elem_for_namespace(ds_ctx, ns_map); if (NULL == elem) { /* Shouldn't happen! */ rc = PMIX_ERR_FATAL; @@ -2521,7 +2042,7 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, /* need to update tracker: * attach to shared memory regions for this namespace and store its info locally * to operate with address and detach/unlink afterwards. */ - rc = _update_ns_elem(elem, ns_info); + rc = _update_ns_elem(ds_ctx, elem, ns_info); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto done; @@ -2531,9 +2052,21 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, meta_seg = elem->meta_seg; data_seg = elem->data_seg; + if( NULL != key ) { + keyhash = PMIX_DS_KEY_HASH(ds_ctx, key); + } + + /* all segment data updated, ctx lock may released */ + if (lock_is_set) { + lock_is_set = false; + if (0 != (rc = pthread_mutex_unlock(&ds_ctx->lock))) { + goto error; + } + } + while (nprocs--) { /* Get the rank meta info in the shared meta segment. */ - rinfo = _get_rank_meta_info(cur_rank, meta_seg); + rinfo = _get_rank_meta_info(ds_ctx, cur_rank, meta_seg); if (NULL == rinfo) { PMIX_OUTPUT_VERBOSE((7, pmix_gds_base_framework.framework_output, "%s:%d:%s: no data for this rank is found in the shared memory. rank %u", @@ -2541,7 +2074,7 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, all_ranks_found = false; continue; } - addr = _get_data_region_by_offset(data_seg, rinfo->offset); + addr = _get_data_region_by_offset(ds_ctx, data_seg, rinfo->offset); if (NULL == addr) { /* This means that meta-info is broken - error is fatal */ rc = PMIX_ERR_FATAL; @@ -2593,22 +2126,22 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, * EXTENSION slot which has key = EXTENSION_SLOT and a size_t value for offset * to next data address for this process. */ - if (0 == strncmp(ESH_KNAME_PTR(addr), ESH_REGION_INVALIDATED, ESH_KNAME_LEN(ESH_REGION_INVALIDATED))) { + if (PMIX_DS_KEY_IS_INVALID(ds_ctx, addr)) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, skip %s region", __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_REGION_INVALIDATED)); /* skip it * go to next item, updating address */ - addr += ESH_KV_SIZE(addr); - } else if (0 == strncmp(ESH_KNAME_PTR(addr), ESH_REGION_EXTENSION, ESH_KNAME_LEN(ESH_REGION_EXTENSION))) { + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); + } else if (PMIX_DS_KEY_IS_EXTSLOT(ds_ctx, addr)) { size_t offset; - memcpy(&offset, ESH_DATA_PTR(addr), sizeof(size_t)); + memcpy(&offset, PMIX_DS_DATA_PTR(ds_ctx, addr), sizeof(size_t)); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, reached %s with %lu value", __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_REGION_EXTENSION, offset)); if (0 < offset) { /* go to next item, updating address */ - addr = _get_data_region_by_offset(data_seg, offset); + addr = _get_data_region_by_offset(ds_ctx, data_seg, offset); if (NULL == addr) { /* This shouldn't happen - error is fatal */ rc = PMIX_ERR_FATAL; @@ -2625,21 +2158,22 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } else if (NULL == key) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, found target key %s", - __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_KNAME_PTR(addr))); + __FILE__, __LINE__, __func__, nspace, cur_rank, PMIX_DS_KNAME_PTR(ds_ctx, addr))); - uint8_t *data_ptr = ESH_DATA_PTR(addr); - size_t data_size = ESH_DATA_SIZE(addr, data_ptr); + uint8_t *data_ptr = PMIX_DS_DATA_PTR(ds_ctx, addr); + size_t data_size = PMIX_DS_DATA_SIZE(ds_ctx, addr, data_ptr); PMIX_CONSTRUCT(&buffer, pmix_buffer_t); - PMIX_LOAD_BUFFER(_client_peer(), &buffer, data_ptr, data_size); + PMIX_LOAD_BUFFER(_client_peer(ds_ctx), &buffer, data_ptr, data_size); int cnt = 1; /* unpack value for this key from the buffer. */ PMIX_VALUE_CONSTRUCT(&val); - PMIX_BFROPS_UNPACK(rc, _client_peer(), &buffer, &val, &cnt, PMIX_VALUE); + PMIX_BFROPS_UNPACK(rc, _client_peer(ds_ctx), &buffer, &val, &cnt, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto done; } - strncpy(info[kval_cnt - 1].key, ESH_KNAME_PTR(addr), ESH_KNAME_LEN((char *)addr)); + pmix_strncpy(info[kval_cnt - 1].key, PMIX_DS_KNAME_PTR(ds_ctx, addr), + PMIX_DS_KNAME_LEN(ds_ctx, addr)); pmix_value_xfer(&info[kval_cnt - 1].value, &val); PMIX_VALUE_DESTRUCT(&val); buffer.base_ptr = NULL; @@ -2648,30 +2182,24 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, key_found = true; kval_cnt--; - addr += ESH_KV_SIZE(addr); - } else if (0 == strncmp(ESH_KNAME_PTR(addr), key, ESH_KNAME_LEN(key))) { + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); + } else if (PMIX_DS_KEY_MATCH(ds_ctx, addr, key, keyhash)) { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, found target key %s", __FILE__, __LINE__, __func__, nspace, cur_rank, key)); /* target key is found, get value */ - uint8_t *data_ptr = ESH_DATA_PTR(addr); - size_t data_size = ESH_DATA_SIZE(addr, data_ptr); + uint8_t *data_ptr = PMIX_DS_DATA_PTR(ds_ctx, addr); + size_t data_size = PMIX_DS_DATA_SIZE(ds_ctx, addr, data_ptr); PMIX_CONSTRUCT(&buffer, pmix_buffer_t); - PMIX_LOAD_BUFFER(_client_peer(), &buffer, data_ptr, data_size); + PMIX_LOAD_BUFFER(_client_peer(ds_ctx), &buffer, data_ptr, data_size); int cnt = 1; /* unpack value for this key from the buffer. */ - PMIX_VALUE_CONSTRUCT(&val); - PMIX_BFROPS_UNPACK(rc, _client_peer(), &buffer, &val, &cnt, PMIX_VALUE); + *kvs = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + PMIX_BFROPS_UNPACK(rc, _client_peer(ds_ctx), &buffer, (void*)*kvs, &cnt, PMIX_VALUE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); goto done; } - PMIX_BFROPS_COPY(rc, _client_peer(), (void**)kvs, &val, PMIX_VALUE); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - goto done; - } - PMIX_VALUE_DESTRUCT(&val); buffer.base_ptr = NULL; buffer.bytes_used = 0; PMIX_DESTRUCT(&buffer); @@ -2680,9 +2208,10 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, } else { PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s: for rank %s:%u, skip key %s look for key %s", - __FILE__, __LINE__, __func__, nspace, cur_rank, ESH_KNAME_PTR(addr), key)); + __FILE__, __LINE__, __func__, nspace, cur_rank, + PMIX_DS_KNAME_PTR(ds_ctx, addr), key)); /* go to next item, updating address */ - addr += ESH_KV_SIZE(addr); + addr += PMIX_DS_KV_SIZE(ds_ctx, addr); kval_cnt--; } } @@ -2694,10 +2223,16 @@ static pmix_status_t _dstore_fetch(const char *nspace, pmix_rank_t rank, done: /* unset lock */ - if (PMIX_SUCCESS != (lock_rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { + lock_rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, rd_unlock); + if (PMIX_SUCCESS != lock_rc) { PMIX_ERROR_LOG(lock_rc); } + /* unset ds_ctx lock */ + if (lock_is_set) { + pthread_mutex_unlock(&ds_ctx->lock); + } + if( rc != PMIX_SUCCESS ){ if ((NULL == key) && (kval_cnt > 0)) { if( NULL != info ) { @@ -2724,13 +2259,21 @@ done: } rc = PMIX_ERR_NOT_FOUND; return rc; + +error: + if (lock_is_set) { + pthread_mutex_unlock(&ds_ctx->lock); + } + PMIX_ERROR_LOG(rc); + return rc; } -static pmix_status_t dstore_fetch(const pmix_proc_t *proc, - pmix_scope_t scope, bool copy, - const char *key, - pmix_info_t info[], size_t ninfo, - pmix_list_t *kvs) +PMIX_EXPORT pmix_status_t pmix_common_dstor_fetch(pmix_common_dstore_ctx_t *ds_ctx, + const pmix_proc_t *proc, + pmix_scope_t scope, bool copy, + const char *key, + pmix_info_t info[], size_t ninfo, + pmix_list_t *kvs) { pmix_kval_t *kv; pmix_value_t *val; @@ -2739,7 +2282,7 @@ static pmix_status_t dstore_fetch(const pmix_proc_t *proc, pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "gds: dstore fetch `%s`", key == NULL ? "NULL" : key); - rc = _dstore_fetch(proc->nspace, proc->rank, key, &val); + rc = _dstore_fetch(ds_ctx, proc->nspace, proc->rank, key, &val); if (PMIX_SUCCESS == rc) { if( NULL == key ) { pmix_info_t *info; @@ -2787,7 +2330,8 @@ static pmix_status_t dstore_fetch(const pmix_proc_t *proc, return rc; } -static pmix_status_t dstore_setup_fork(const pmix_proc_t *peer, char ***env) +PMIX_EXPORT pmix_status_t pmix_common_dstor_setup_fork(pmix_common_dstore_ctx_t *ds_ctx, const char *base_path_env, + const pmix_proc_t *peer, char ***env) { pmix_status_t rc = PMIX_SUCCESS; ns_map_data_t *ns_map = NULL; @@ -2795,41 +2339,43 @@ static pmix_status_t dstore_setup_fork(const pmix_proc_t *peer, char ***env) pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "gds: dstore setup fork"); - if (NULL == _esh_session_map_search) { + if (NULL == ds_ctx->session_map_search) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); return rc; } - if (NULL == (ns_map = _esh_session_map_search(peer->nspace))) { + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, peer->nspace))) { rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); return rc; } - if ((NULL == _base_path) || (strlen(_base_path) == 0)){ + if ((NULL == ds_ctx->base_path) || (strlen(ds_ctx->base_path) == 0)){ rc = PMIX_ERR_NOT_AVAILABLE; PMIX_ERROR_LOG(rc); return rc; } - if(PMIX_SUCCESS != (rc = pmix_setenv(PMIX_DSTORE_ESH_BASE_PATH, - _ESH_SESSION_path(ns_map->tbl_idx), true, env))){ + if(PMIX_SUCCESS != (rc = pmix_setenv(base_path_env, + _ESH_SESSION_path(ds_ctx->session_array, ns_map->tbl_idx), + true, env))){ PMIX_ERROR_LOG(rc); } + return rc; } -static pmix_status_t dstore_add_nspace(const char *nspace, - pmix_info_t info[], - size_t ninfo) +PMIX_EXPORT pmix_status_t pmix_common_dstor_add_nspace(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, pmix_info_t info[], size_t ninfo) { - pmix_status_t rc; + pmix_status_t rc = PMIX_SUCCESS; size_t tbl_idx=0; - uid_t jobuid = _jobuid; - char setjobuid = _setjobuid; + uid_t jobuid = ds_ctx->jobuid; + char setjobuid = ds_ctx->setjobuid; size_t n; ns_map_data_t *ns_map = NULL; + uint32_t local_size = 0; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "gds: dstore add nspace"); @@ -2841,31 +2387,36 @@ static pmix_status_t dstore_add_nspace(const char *nspace, setjobuid = 1; continue; } + if (0 == strcmp(PMIX_LOCAL_SIZE, info[n].key)) { + local_size = info[n].value.data.uint32; + continue; + } } } - if (PMIX_SUCCESS != _esh_jobuid_tbl_search(jobuid, &tbl_idx)) { + if (PMIX_SUCCESS != _esh_jobuid_tbl_search(ds_ctx, jobuid, &tbl_idx)) { - rc = _esh_session_tbl_add(&tbl_idx); + rc = _esh_session_tbl_add(ds_ctx, &tbl_idx); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } - ns_map = _esh_session_map(nspace, tbl_idx); + ns_map = _esh_session_map(ds_ctx, nspace, local_size, tbl_idx); if (NULL == ns_map) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); return rc; } - if (PMIX_SUCCESS != (rc =_esh_session_init(tbl_idx, ns_map, jobuid, setjobuid))) { + if (PMIX_SUCCESS != (rc =_esh_session_init(ds_ctx, tbl_idx, ns_map, + local_size, jobuid, setjobuid))) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); return rc; } } else { - ns_map = _esh_session_map(nspace, tbl_idx); + ns_map = _esh_session_map(ds_ctx, nspace, local_size, tbl_idx); if (NULL == ns_map) { rc = PMIX_ERROR; PMIX_ERROR_LOG(rc); @@ -2873,10 +2424,19 @@ static pmix_status_t dstore_add_nspace(const char *nspace, } } + /* lock init */ + ds_ctx->lock_cbs->init(&_ESH_SESSION_lock(ds_ctx->session_array, tbl_idx), + ds_ctx->base_path, nspace, local_size, ds_ctx->jobuid, + ds_ctx->setjobuid); + if (NULL == _ESH_SESSION_lock(ds_ctx->session_array, tbl_idx)) { + PMIX_ERROR_LOG(rc); + return rc; + } + return PMIX_SUCCESS; } -static pmix_status_t dstore_del_nspace(const char* nspace) +PMIX_EXPORT pmix_status_t pmix_common_dstor_del_nspace(pmix_common_dstore_ctx_t *ds_ctx, const char* nspace) { pmix_status_t rc = PMIX_SUCCESS; size_t map_idx, size; @@ -2891,20 +2451,20 @@ static pmix_status_t dstore_del_nspace(const char* nspace) PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s delete nspace `%s`", __FILE__, __LINE__, __func__, nspace)); - if (NULL == (ns_map_data = _esh_session_map_search(nspace))) { + if (NULL == (ns_map_data = ds_ctx->session_map_search(ds_ctx, nspace))) { rc = PMIX_ERR_NOT_AVAILABLE; return rc; } dstor_track_idx = ns_map_data->track_idx; session_tbl_idx = ns_map_data->tbl_idx; - size = pmix_value_array_get_size(_ns_map_array); - ns_map = PMIX_VALUE_ARRAY_GET_BASE(_ns_map_array, ns_map_t); + size = pmix_value_array_get_size(ds_ctx->ns_map_array); + ns_map = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->ns_map_array, ns_map_t); for (map_idx = 0; map_idx < size; map_idx++){ if (ns_map[map_idx].in_use && (ns_map[map_idx].data.tbl_idx == ns_map_data->tbl_idx)) { if (0 == strcmp(ns_map[map_idx].data.name, nspace)) { - _esh_session_map_clean(&ns_map[map_idx]); + _esh_session_map_clean(ds_ctx, &ns_map[map_idx]); continue; } in_use++; @@ -2914,68 +2474,29 @@ static pmix_status_t dstore_del_nspace(const char* nspace) /* A lot of nspaces may be using same session info * session record can only be deleted once all references are gone */ if (!in_use) { - session_tbl = PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t); + session_tbl = PMIX_VALUE_ARRAY_GET_BASE(ds_ctx->session_array, session_t); PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, "%s:%d:%s delete session for jobuid: %d", __FILE__, __LINE__, __func__, session_tbl[session_tbl_idx].jobuid)); - size = pmix_value_array_get_size(_ns_track_array); + size = pmix_value_array_get_size(ds_ctx->ns_track_array); if (size && (dstor_track_idx >= 0)) { - if((dstor_track_idx + 1) > size) { + if((dstor_track_idx + 1) > (int)size) { rc = PMIX_ERR_VALUE_OUT_OF_BOUNDS; PMIX_ERROR_LOG(rc); goto exit; } - trk = pmix_value_array_get_item(_ns_track_array, dstor_track_idx); + trk = pmix_value_array_get_item(ds_ctx->ns_track_array, dstor_track_idx); if (true == trk->in_use) { PMIX_DESTRUCT(trk); + pmix_value_array_remove_item(ds_ctx->ns_track_array, dstor_track_idx); } } - _esh_session_release(&session_tbl[session_tbl_idx]); + _esh_session_release(ds_ctx, session_tbl_idx); } exit: return rc; } -static pmix_status_t dstore_assign_module(pmix_info_t *info, size_t ninfo, - int *priority) -{ - size_t n, m; - char **options; - - *priority = 20; - if (NULL != info) { - for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { - options = pmix_argv_split(info[n].value.data.string, ','); - for (m=0; NULL != options[m]; m++) { - if (0 == strcmp(options[m], "ds12")) { - /* they specifically asked for us */ - *priority = 100; - break; - } - if (0 == strcmp(options[m], "dstore")) { - /* they are asking for any dstore module - we - * take an intermediate priority in case another - * dstore is more modern than us */ - *priority = 50; - break; - } - } - pmix_argv_free(options); - break; - } - } - } - -#if 0 - if PMIX_GDS_MODULE != "ds12" - *proirity = 0; - else PMIX_GDS_MODULE == "ds12" || !PMIX_GDS_MODULE - *priority = -1; -#endif - return PMIX_SUCCESS; -} - static inline int _my_client(const char *nspace, pmix_rank_t rank) { pmix_peer_t *peer; @@ -2998,16 +2519,59 @@ static inline int _my_client(const char *nspace, pmix_rank_t rank) * host has received data from some other peer. It therefore * always contains data solely from remote procs, and we * shall store it accordingly */ -static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace, - pmix_list_t *cbs, - pmix_byte_object_t *bo) +PMIX_EXPORT pmix_status_t pmix_common_dstor_store_modex(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *buf) { - pmix_nspace_t *ns = (pmix_nspace_t*)nspace; + pmix_status_t rc = PMIX_SUCCESS; + pmix_status_t rc1 = PMIX_SUCCESS; + pmix_namespace_t *ns = (pmix_namespace_t*)nspace; + ns_map_data_t *ns_map; + + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, ns->nspace))) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + return rc; + } + + /* set exclusive lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_lock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + + rc = pmix_gds_base_store_modex(nspace, cbs, buf, (pmix_gds_base_store_modex_cb_fn_t)_dstor_store_modex_cb, ds_ctx); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + + /* unset lock */ + rc1 = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_unlock); + if (PMIX_SUCCESS != rc1) { + PMIX_ERROR_LOG(rc1); + if (PMIX_SUCCESS == rc) { + rc = rc1; + } + } + + return rc; +} + +static pmix_status_t _dstor_store_modex_cb(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_byte_object_t *bo) +{ + pmix_namespace_t *ns = (pmix_namespace_t*)nspace; pmix_status_t rc = PMIX_SUCCESS; int32_t cnt; pmix_buffer_t pbkt; pmix_proc_t proc; pmix_kval_t *kv; + ns_map_data_t *ns_map; + pmix_buffer_t tmp; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "[%s:%d] gds:dstore:store_modex for nspace %s", @@ -3048,6 +2612,10 @@ static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace, PMIX_DESTRUCT(&pbkt); return PMIX_SUCCESS; } + + /* Prepare a buffer to be provided to the dstor store primitive */ + PMIX_CONSTRUCT(&tmp, pmix_buffer_t); + /* unpack the remaining values until we hit the end of the buffer */ cnt = 1; kv = PMIX_NEW(pmix_kval_t); @@ -3063,29 +2631,68 @@ static pmix_status_t dstore_store_modex(struct pmix_nspace_t *nspace, PMIX_DESTRUCT(&pbkt); return rc; } - if (PMIX_SUCCESS != (rc = dstore_store(&proc, PMIX_REMOTE, kv))) { - PMIX_ERROR_LOG(rc); - } - PMIX_RELEASE(kv); // maintain accounting as the hash increments the ref count - /* continue along */ + + /* place the key to the to be provided to _dstore_store_nolock */ + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &tmp, kv, 1, PMIX_KVAL); + + /* Release the kv to maintain accounting + * as the hash increments the ref count */ + PMIX_RELEASE(kv); + + /* proceed to the next element */ kv = PMIX_NEW(pmix_kval_t); cnt = 1; PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &pbkt, kv, &cnt, PMIX_KVAL); } - PMIX_RELEASE(kv); // maintain accounting + + /* Release the kv that didn't received the value + * because input buffer was exhausted */ + PMIX_RELEASE(kv); if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { PMIX_ERROR_LOG(rc); } else { rc = PMIX_SUCCESS; } + + /* Create a key-value pair with the buffer + * to be passed to _dstore_store_nolock */ + kv = PMIX_NEW(pmix_kval_t); + PMIX_VALUE_CREATE(kv->value, 1); + kv->value->type = PMIX_BYTE_OBJECT; + PMIX_UNLOAD_BUFFER(&tmp, kv->value->data.bo.bytes, kv->value->data.bo.size); + + /* Get the namespace map element for the process "proc" */ + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, proc.nspace))) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + bo->bytes = pbkt.base_ptr; + bo->size = pbkt.bytes_used; // restore the incoming data + pbkt.base_ptr = NULL; + PMIX_DESTRUCT(&pbkt); + return rc; + } + + /* Store all keys at once */ + rc = _dstore_store_nolock(ds_ctx, ns_map, proc.rank, kv); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + + /* Release all resources */ + PMIX_RELEASE(kv); + PMIX_DESTRUCT(&tmp); + + /* Reset the input buffer */ bo->bytes = pbkt.base_ptr; - bo->size = pbkt.bytes_used; // restore the incoming data + bo->size = pbkt.bytes_used; pbkt.base_ptr = NULL; PMIX_DESTRUCT(&pbkt); + return rc; } -static pmix_status_t _store_job_info(pmix_proc_t *proc) +static pmix_status_t _store_job_info(pmix_common_dstore_ctx_t *ds_ctx, ns_map_data_t *ns_map, + pmix_proc_t *proc) { pmix_cb_t cb; pmix_kval_t *kv; @@ -3113,7 +2720,7 @@ static pmix_status_t _store_job_info(pmix_proc_t *proc) } PMIX_LIST_FOREACH(kv, &cb.kvs, pmix_kval_t) { - if ((PMIX_PROC_IS_V1(_client_peer()) || PMIX_PROC_IS_V20(_client_peer())) && + if ((PMIX_PROC_IS_V1(_client_peer(ds_ctx)) || PMIX_PROC_IS_V20(_client_peer(ds_ctx))) && 0 != strncmp("pmix.", kv->key, 4) && kv->value->type == PMIX_DATA_ARRAY) { pmix_info_t *info; @@ -3150,7 +2757,7 @@ static pmix_status_t _store_job_info(pmix_proc_t *proc) } PMIX_UNLOAD_BUFFER(&buf, kvp->value->data.bo.bytes, kvp->value->data.bo.size); - if (PMIX_SUCCESS != (rc = _dstore_store(proc->nspace, proc->rank, kvp))) { + if (PMIX_SUCCESS != (rc = _dstore_store_nolock(ds_ctx, ns_map, proc->rank, kvp))) { PMIX_ERROR_LOG(rc); goto exit; } @@ -3162,11 +2769,12 @@ exit: return rc; } -static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, - pmix_buffer_t *reply) +PMIX_EXPORT pmix_status_t pmix_common_dstor_register_job_info(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_peer_t *pr, + pmix_buffer_t *reply) { pmix_peer_t *peer = (pmix_peer_t*)pr; - pmix_nspace_t *ns = peer->nptr; + pmix_namespace_t *ns = peer->nptr; char *msg; pmix_status_t rc; pmix_proc_t proc; @@ -3178,10 +2786,25 @@ static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, peer->info->pname.nspace, peer->info->pname.rank); if (0 == ns->ndelivered) { // don't store twice - _client_compat_save(peer); - (void)strncpy(proc.nspace, ns->nspace, PMIX_MAX_NSLEN); + ns_map_data_t *ns_map; + + _client_compat_save(ds_ctx, peer); + pmix_strncpy(proc.nspace, ns->nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; - rc = _store_job_info(&proc); + if (NULL == (ns_map = ds_ctx->session_map_search(ds_ctx, proc.nspace))) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + return rc; + } + + /* set exclusive lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_lock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + + rc = _store_job_info(ds_ctx, ns_map, &proc); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; @@ -3189,12 +2812,18 @@ static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, for (rank=0; rank < ns->nprocs; rank++) { proc.rank = rank; - rc = _store_job_info(&proc); + rc = _store_job_info(ds_ctx, ns_map, &proc); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); return rc; } } + /* unset lock */ + rc = _ESH_LOCK(ds_ctx, ns_map->tbl_idx, wr_unlock); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } } /* answer to client */ @@ -3208,7 +2837,9 @@ static pmix_status_t dstore_register_job_info(struct pmix_peer_t *pr, return rc; } -static pmix_status_t dstore_store_job_info(const char *nspace, pmix_buffer_t *buf) +PMIX_EXPORT pmix_status_t pmix_common_dstor_store_job_info(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, + pmix_buffer_t *job_data) { pmix_status_t rc = PMIX_SUCCESS; @@ -3217,7 +2848,7 @@ static pmix_status_t dstore_store_job_info(const char *nspace, pmix_buffer_t *b pmix_globals.myid.nspace, pmix_globals.myid.rank, nspace); /* check buf data */ - if ((NULL == buf) || (0 == buf->bytes_used)) { + if ((NULL == job_data) || (0 == job_data->bytes_used)) { rc = PMIX_ERR_BAD_PARAM; PMIX_ERROR_LOG(rc); return rc; @@ -3225,23 +2856,23 @@ static pmix_status_t dstore_store_job_info(const char *nspace, pmix_buffer_t *b return rc; } -static void _client_compat_save(pmix_peer_t *peer) +static void _client_compat_save(pmix_common_dstore_ctx_t *ds_ctx, pmix_peer_t *peer) { - pmix_nspace_t *nptr = NULL; + pmix_namespace_t *nptr = NULL; - if (NULL == _clients_peer) { - _clients_peer = PMIX_NEW(pmix_peer_t); - nptr = PMIX_NEW(pmix_nspace_t); - _clients_peer->nptr = nptr; + if (NULL == ds_ctx->clients_peer) { + ds_ctx->clients_peer = PMIX_NEW(pmix_peer_t); + nptr = PMIX_NEW(pmix_namespace_t); + ds_ctx->clients_peer->nptr = nptr; } - _clients_peer->nptr->compat = peer->nptr->compat; - _clients_peer->proc_type = peer->proc_type; + ds_ctx->clients_peer->nptr->compat = peer->nptr->compat; + ds_ctx->clients_peer->proc_type = peer->proc_type; } -static inline pmix_peer_t * _client_peer(void) +static inline pmix_peer_t * _client_peer(pmix_common_dstore_ctx_t *ds_ctx) { - if (NULL == _clients_peer) { + if (NULL == ds_ctx->clients_peer) { return pmix_globals.mypeer; } - return _clients_peer; + return ds_ctx->clients_peer; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_base.h new file mode 100644 index 0000000000..7989ae6ca5 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_base.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_DSTORE_H +#define PMIX_DSTORE_H + +#include + + +#include "src/mca/gds/gds.h" +#include "src/mca/pshmem/pshmem.h" + +BEGIN_C_DECLS + +#include +#include "src/class/pmix_value_array.h" +#include "dstore_common.h" +#include "dstore_segment.h" +#include "dstore_file.h" + +#define INITIAL_SEG_SIZE 4096 +#define NS_META_SEG_SIZE (1<<22) +#define NS_DATA_SEG_SIZE (1<<22) + +#define PMIX_DSTORE_ESH_BASE_PATH "PMIX_DSTORE_ESH_BASE_PATH" +#define PMIX_DSTORE_VER_BASE_PATH_FMT "PMIX_DSTORE_%d_BASE_PATH" + +typedef struct ns_map_data_s ns_map_data_t; +typedef struct session_s session_t; +typedef struct ns_map_s ns_map_t; + +typedef ns_map_data_t * (*session_map_search_fn_t)(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace); + +struct pmix_common_dstore_ctx_s { + char *ds_name; + char *base_path; + uid_t jobuid; + char setjobuid; + + pmix_value_array_t *session_array; + pmix_value_array_t *ns_map_array; + pmix_value_array_t *ns_track_array; + + pmix_common_lock_callbacks_t *lock_cbs; + pmix_common_dstore_file_cbs_t *file_cbs; + + size_t initial_segment_size; + size_t meta_segment_size; + size_t data_segment_size; + size_t lock_segment_size; + + size_t max_ns_num; + size_t max_meta_elems; + + session_map_search_fn_t session_map_search; + pmix_peer_t *clients_peer; + /* If _direct_mode is set, it means that we use linear search + * along the array of rank meta info objects inside a meta segment + * to find the requested rank. Otherwise, we do a fast lookup + * based on rank and directly compute offset. + * This mode is called direct because it's effectively used in + * sparse communication patterns when direct modex is usually used. + */ + int direct_mode; + /* dstore ctx protect lock, uses for clients only */ + pthread_mutex_t lock; +}; + +struct session_s { + int in_use; + uid_t jobuid; + char setjobuid; + char *nspace_path; + pmix_dstore_seg_desc_t *sm_seg_first; + pmix_dstore_seg_desc_t *sm_seg_last; + pmix_common_dstor_lock_ctx_t lock; +}; + +struct ns_map_data_s { + char name[PMIX_MAX_NSLEN+1]; + size_t tbl_idx; + int track_idx; +}; + +struct ns_map_s { + int in_use; + ns_map_data_t data; +}; + +/* initial segment format: + * size_t num_elems; + * size_t full; //indicate to client that it needs to attach to the next segment + * ns_seg_info_t ns_seg_info[max_ns_num]; + */ + +typedef struct { + ns_map_data_t ns_map; + size_t num_meta_seg;/* read by clients to attach to this number of segments. */ + size_t num_data_seg; +} ns_seg_info_t; + +/* meta segment format: + * size_t num_elems; + * rank_meta_info meta_info[max_meta_elems]; + */ + +typedef struct { + size_t rank; + size_t offset; + size_t count; +} rank_meta_info; + +typedef struct { + pmix_value_array_t super; + ns_map_data_t ns_map; + size_t num_meta_seg; + size_t num_data_seg; + pmix_dstore_seg_desc_t *meta_seg; + pmix_dstore_seg_desc_t *data_seg; + bool in_use; +} ns_track_elem_t; + +typedef struct { + pmix_list_item_t super; + pmix_common_dstor_lock_ctx_t *lock; +} lock_track_item_t; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_common.h b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_common.h new file mode 100644 index 0000000000..466eccd9a5 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_common.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_GDS_DS_BASE_H_ +#define PMIX_GDS_DS_BASE_H_ + +#include +#include +#include + +#include "src/include/pmix_globals.h" +#include "src/class/pmix_list.h" +#include "src/mca/gds/gds.h" +#include "src/mca/pshmem/pshmem.h" +#include "src/mca/common/dstore/dstore_file.h" + +typedef void* pmix_common_dstor_lock_ctx_t; + +typedef pmix_status_t (*pmix_common_dstor_lock_init_fn_t)(pmix_common_dstor_lock_ctx_t *ctx, + const char *base_path, const char *name, + uint32_t local_size, uid_t uid, bool setuid); +typedef void (*pmix_common_dstor_lock_finalize_fn_t)(pmix_common_dstor_lock_ctx_t *ctx); +typedef pmix_status_t (*pmix_common_dstor_lock_rd_get_fn_t)(pmix_common_dstor_lock_ctx_t ctx); +typedef pmix_status_t (*pmix_common_dstor_lock_rd_rel_fn_t)(pmix_common_dstor_lock_ctx_t ctx); +typedef pmix_status_t (*pmix_common_dstor_lock_wr_get_fn_t)(pmix_common_dstor_lock_ctx_t ctx); +typedef pmix_status_t (*pmix_common_dstor_lock_wr_rel_fn_t)(pmix_common_dstor_lock_ctx_t ctx); + +typedef struct { + pmix_common_dstor_lock_init_fn_t init; + pmix_common_dstor_lock_finalize_fn_t finalize; + pmix_common_dstor_lock_rd_get_fn_t rd_lock; + pmix_common_dstor_lock_rd_rel_fn_t rd_unlock; + pmix_common_dstor_lock_wr_get_fn_t wr_lock; + pmix_common_dstor_lock_wr_rel_fn_t wr_unlock; +} pmix_common_lock_callbacks_t; + +typedef struct pmix_common_dstore_ctx_s pmix_common_dstore_ctx_t; + +PMIX_EXPORT pmix_common_dstore_ctx_t *pmix_common_dstor_init(const char *ds_name, pmix_info_t info[], size_t ninfo, + pmix_common_lock_callbacks_t *lock_cb, + pmix_common_dstore_file_cbs_t *file_cb); +PMIX_EXPORT void pmix_common_dstor_finalize(pmix_common_dstore_ctx_t *ds_ctx); +PMIX_EXPORT pmix_status_t pmix_common_dstor_add_nspace(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, pmix_info_t info[], size_t ninfo); +PMIX_EXPORT pmix_status_t pmix_common_dstor_del_nspace(pmix_common_dstore_ctx_t *ds_ctx, const char* nspace); +PMIX_EXPORT pmix_status_t pmix_common_dstor_setup_fork(pmix_common_dstore_ctx_t *ds_ctx, const char *base_path_env, + const pmix_proc_t *peer, char ***env); +PMIX_EXPORT pmix_status_t pmix_common_dstor_cache_job_info(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *ns, + pmix_info_t info[], size_t ninfo); +PMIX_EXPORT pmix_status_t pmix_common_dstor_register_job_info(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_peer_t *pr, + pmix_buffer_t *reply); +PMIX_EXPORT pmix_status_t pmix_common_dstor_store_job_info(pmix_common_dstore_ctx_t *ds_ctx, + const char *nspace, + pmix_buffer_t *job_data); +PMIX_EXPORT pmix_status_t pmix_common_dstor_store(pmix_common_dstore_ctx_t *ds_ctx, + const pmix_proc_t *proc, + pmix_scope_t scope, + pmix_kval_t *kv); +PMIX_EXPORT pmix_status_t pmix_common_dstor_fetch(pmix_common_dstore_ctx_t *ds_ctx, + const pmix_proc_t *proc, + pmix_scope_t scope, bool copy, + const char *key, + pmix_info_t info[], size_t ninfo, + pmix_list_t *kvs); +PMIX_EXPORT pmix_status_t pmix_common_dstor_store_modex(pmix_common_dstore_ctx_t *ds_ctx, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *buff); +#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_file.h b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_file.h new file mode 100644 index 0000000000..576149b056 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_file.h @@ -0,0 +1,161 @@ +#ifndef DSTORE_FORMAT_H +#define DSTORE_FORMAT_H + +typedef size_t (*pmix_common_dstore_kv_size_fn)(uint8_t *addr); +typedef char* (*pmix_common_dstore_key_name_ptr_fn)(uint8_t *addr); +typedef size_t (*pmix_common_dstore_key_name_len_fn)(char *key); +typedef uint8_t* (*pmix_common_dstore_data_ptr_fn)(uint8_t *addr); +typedef size_t (*pmix_common_dstore_data_size_fn)(uint8_t *addr, uint8_t* data_ptr); +typedef size_t (*pmix_common_dstore_key_size_fn)(char *key, size_t data_size); +typedef size_t (*pmix_common_dstore_ext_slot_size_fn)(void); +typedef int (*pmix_common_dstore_put_key_fn)(uint8_t *addr, char *key, void *buf, + size_t size); +typedef bool (*pmix_common_dstore_is_invalid_fn)(uint8_t *addr); +typedef bool (*pmix_common_dstore_is_extslot_fn)(uint8_t *addr); +typedef void (*pmix_common_dstore_set_invalid_fn)(uint8_t *addr); +typedef size_t (*pmix_common_dstore_key_hash_fn)(const char *key); +typedef bool (*pmix_common_dstore_key_match_fn)(uint8_t *addr, const char *key, + size_t key_hash); + +typedef struct { + const char *name; + pmix_common_dstore_kv_size_fn kval_size; + pmix_common_dstore_key_name_ptr_fn kname_ptr; + pmix_common_dstore_key_name_len_fn kname_len; + pmix_common_dstore_data_ptr_fn data_ptr; + pmix_common_dstore_data_size_fn data_size; + pmix_common_dstore_key_size_fn key_size; + pmix_common_dstore_ext_slot_size_fn ext_slot_size; + pmix_common_dstore_put_key_fn put_key; + pmix_common_dstore_is_invalid_fn is_invalid; + pmix_common_dstore_is_extslot_fn is_extslot; + pmix_common_dstore_set_invalid_fn set_invalid; + pmix_common_dstore_key_hash_fn key_hash; + pmix_common_dstore_key_match_fn key_match; +} pmix_common_dstore_file_cbs_t; + +#define ESH_REGION_EXTENSION "EXTENSION_SLOT" +#define ESH_REGION_INVALIDATED "INVALIDATED" +#define ESH_ENV_INITIAL_SEG_SIZE "INITIAL_SEG_SIZE" +#define ESH_ENV_NS_META_SEG_SIZE "NS_META_SEG_SIZE" +#define ESH_ENV_NS_DATA_SEG_SIZE "NS_DATA_SEG_SIZE" +#define ESH_ENV_LINEAR "SM_USE_LINEAR_SEARCH" + +#define ESH_MIN_KEY_LEN (sizeof(ESH_REGION_INVALIDATED)) + +#define PMIX_DS_PUT_KEY(rc, ctx, addr, key, buf, size) \ + do { \ + rc = PMIX_ERROR; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->put_key) { \ + rc = (ctx)->file_cbs->put_key(addr, key, buf, size); \ + } \ + } while(0) + +#define PMIX_DS_KV_SIZE(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + size_t size = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->kval_size) { \ + size = (ctx)->file_cbs->kval_size(addr); \ + } \ + size; \ +}) + +#define PMIX_DS_KNAME_PTR(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + char *name_ptr = NULL; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->kname_ptr) { \ + name_ptr = (ctx)->file_cbs->kname_ptr(addr); \ + } \ + name_ptr; \ +}) + +#define PMIX_DS_KNAME_LEN(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + size_t len = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->kname_len) { \ + len = (ctx)->file_cbs->kname_len((char*)addr); \ + } \ + len; \ +}) + +#define PMIX_DS_DATA_PTR(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + uint8_t *data_ptr = NULL; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->data_ptr) { \ + data_ptr = (ctx)->file_cbs->data_ptr(addr); \ + } \ + data_ptr; \ +}) + +#define PMIX_DS_DATA_SIZE(ctx, addr, data_ptr) \ +__pmix_attribute_extension__ ({ \ + size_t size = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->data_size) { \ + size = (ctx)->file_cbs->data_size(addr, data_ptr); \ + } \ + size; \ +}) + +#define PMIX_DS_KEY_SIZE(ctx, key, data_size) \ +__pmix_attribute_extension__ ({ \ + size_t __size = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->key_size) { \ + __size = (ctx)->file_cbs->key_size(key, data_size); \ + } \ + __size; \ +}) + +#define PMIX_DS_SLOT_SIZE(ctx) \ +__pmix_attribute_extension__ ({ \ + size_t __size = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->ext_slot_size) { \ + __size = (ctx)->file_cbs->ext_slot_size(); \ + } \ + __size; \ +}) + +#define PMIX_DS_KEY_HASH(ctx, key) \ +__pmix_attribute_extension__ ({ \ + size_t keyhash = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->key_hash) { \ + keyhash = (ctx)->file_cbs->key_hash(key); \ + } \ + keyhash; \ +}) + +#define PMIX_DS_KEY_MATCH(ctx, addr, key, hash) \ +__pmix_attribute_extension__ ({ \ + int ret = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->key_match) { \ + ret = (ctx)->file_cbs->key_match(addr, key, hash); \ + } \ + ret; \ +}) + +#define PMIX_DS_KEY_IS_INVALID(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + int ret = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->is_invalid) { \ + ret = (ctx)->file_cbs->is_invalid(addr); \ + } \ + ret; \ +}) + +#define PMIX_DS_KEY_SET_INVALID(ctx, addr) \ + do { \ + if ((ctx)->file_cbs && (ctx)->file_cbs->set_invalid) { \ + (ctx)->file_cbs->set_invalid(addr); \ + } \ + } while(0) + +#define PMIX_DS_KEY_IS_EXTSLOT(ctx, addr) \ +__pmix_attribute_extension__ ({ \ + int ret = 0; \ + if ((ctx)->file_cbs && (ctx)->file_cbs->is_invalid) { \ + ret = (ctx)->file_cbs->is_extslot(addr); \ + } \ + ret; \ +}) + + +#endif // DSTORE_FORMAT_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_segment.c b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_segment.c new file mode 100644 index 0000000000..a219bed9c0 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_segment.c @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif + +#ifdef HAVE_SYS_AUXV_H +#include +#endif + +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" +#include "src/mca/pshmem/base/base.h" +#include "src/util/error.h" +#include "src/util/output.h" + +#include "dstore_common.h" +#include "dstore_segment.h" + +static size_t _initial_segment_size; +static size_t _meta_segment_size; +static size_t _data_segment_size; + +PMIX_EXPORT int pmix_common_dstor_getpagesize(void) +{ +#if defined(_SC_PAGESIZE ) + return sysconf(_SC_PAGESIZE); +#elif defined(_SC_PAGE_SIZE) + return sysconf(_SC_PAGE_SIZE); +#else + return 65536; /* safer to overestimate than under */ +#endif +} + +PMIX_EXPORT size_t pmix_common_dstor_getcacheblocksize(void) +{ + size_t cache_line = 0; + +#if defined(_SC_LEVEL1_DCACHE_LINESIZE) + cache_line = sysconf(_SC_LEVEL1_DCACHE_LINESIZE); +#endif +#if (defined(HAVE_SYS_AUXV_H)) && (defined(AT_DCACHEBSIZE)) + if (0 == cache_line) { + cache_line = getauxval(AT_DCACHEBSIZE); + } +#endif + return cache_line; +} + +PMIX_EXPORT void pmix_common_dstor_init_segment_info(size_t initial_segment_size, + size_t meta_segment_size, + size_t data_segment_size) +{ + _initial_segment_size = initial_segment_size; + _meta_segment_size = meta_segment_size; + _data_segment_size = data_segment_size; +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_create_new_lock_seg(const char *base_path, size_t size, + const char *name, uint32_t id, uid_t uid, bool setuid) +{ + pmix_status_t rc; + char file_name[PMIX_PATH_MAX]; + pmix_dstore_seg_desc_t *new_seg = NULL; + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s: segment type %d, nspace %s, id %u", + __FILE__, __LINE__, __func__, PMIX_DSTORE_NS_LOCK_SEGMENT, + name, id)); + + snprintf(file_name, PMIX_PATH_MAX, "%s/smlockseg-%s", base_path, name); + new_seg = (pmix_dstore_seg_desc_t*)malloc(sizeof(pmix_dstore_seg_desc_t)); + if (new_seg) { + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = PMIX_DSTORE_NS_LOCK_SEGMENT; + rc = pmix_pshmem.segment_create(&new_seg->seg_info, file_name, size); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto err_exit; + } + memset(new_seg->seg_info.seg_base_addr, 0, size); + + if (setuid > 0){ + rc = PMIX_ERR_PERM; + if (0 > chown(file_name, (uid_t) uid, (gid_t) -1)){ + PMIX_ERROR_LOG(rc); + goto err_exit; + } + /* set the mode as required */ + if (0 > chmod(file_name, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP )) { + PMIX_ERROR_LOG(rc); + goto err_exit; + } + } + } + return new_seg; + + err_exit: + if( NULL != new_seg ){ + free(new_seg); + } + return NULL; + +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_attach_new_lock_seg(const char *base_path, + size_t size, const char *name, uint32_t id) +{ + pmix_status_t rc; + pmix_dstore_seg_desc_t *new_seg = NULL; + new_seg = (pmix_dstore_seg_desc_t*)malloc(sizeof(pmix_dstore_seg_desc_t)); + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = PMIX_DSTORE_NS_LOCK_SEGMENT; + new_seg->seg_info.seg_size = size; + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s: segment type %d, name %s, id %u", + __FILE__, __LINE__, __func__, new_seg->type, name, id)); + + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smlockseg-%s", + base_path, name); + rc = pmix_pshmem.segment_attach(&new_seg->seg_info, PMIX_PSHMEM_RW); + if (PMIX_SUCCESS != rc) { + free(new_seg); + new_seg = NULL; + } + return new_seg; +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_create_new_segment(pmix_dstore_segment_type type, + const char *base_path, const char *name, uint32_t id, + uid_t uid, bool setuid) +{ + pmix_status_t rc; + char file_name[PMIX_PATH_MAX]; + size_t size; + pmix_dstore_seg_desc_t *new_seg = NULL; + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s: segment type %d, nspace %s, id %u", + __FILE__, __LINE__, __func__, type, name, id)); + + switch (type) { + case PMIX_DSTORE_INITIAL_SEGMENT: + size = _initial_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/initial-pmix_shared-segment-%u", + base_path, id); + break; + case PMIX_DSTORE_NS_META_SEGMENT: + size = _meta_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/smseg-%s-%u", base_path, name, id); + break; + case PMIX_DSTORE_NS_DATA_SEGMENT: + size = _data_segment_size; + snprintf(file_name, PMIX_PATH_MAX, "%s/smdataseg-%s-%d", base_path, name, id); + break; + default: + PMIX_ERROR_LOG(PMIX_ERROR); + return NULL; + } + new_seg = (pmix_dstore_seg_desc_t*)malloc(sizeof(pmix_dstore_seg_desc_t)); + if (new_seg) { + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = type; + rc = pmix_pshmem.segment_create(&new_seg->seg_info, file_name, size); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto err_exit; + } + memset(new_seg->seg_info.seg_base_addr, 0, size); + + if (setuid > 0){ + rc = PMIX_ERR_PERM; + if (0 > chown(file_name, (uid_t) uid, (gid_t) -1)){ + PMIX_ERROR_LOG(rc); + goto err_exit; + } + /* set the mode as required */ + if (0 > chmod(file_name, S_IRUSR | S_IRGRP | S_IWGRP )) { + PMIX_ERROR_LOG(rc); + goto err_exit; + } + } + } + return new_seg; + +err_exit: + if( NULL != new_seg ){ + free(new_seg); + } + return NULL; +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_attach_new_segment(pmix_dstore_segment_type type, const char *base_path, + const char *name, uint32_t id) +{ + pmix_status_t rc; + pmix_dstore_seg_desc_t *new_seg = NULL; + new_seg = (pmix_dstore_seg_desc_t*)malloc(sizeof(pmix_dstore_seg_desc_t)); + new_seg->id = id; + new_seg->next = NULL; + new_seg->type = type; + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s: segment type %d, nspace %s, id %u", + __FILE__, __LINE__, __func__, type, name, id)); + + switch (type) { + case PMIX_DSTORE_INITIAL_SEGMENT: + new_seg->seg_info.seg_size = _initial_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/initial-pmix_shared-segment-%u", + base_path, id); + break; + case PMIX_DSTORE_NS_META_SEGMENT: + new_seg->seg_info.seg_size = _meta_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smseg-%s-%u", + base_path, name, id); + break; + case PMIX_DSTORE_NS_DATA_SEGMENT: + new_seg->seg_info.seg_size = _data_segment_size; + snprintf(new_seg->seg_info.seg_name, PMIX_PATH_MAX, "%s/smdataseg-%s-%d", + base_path, name, id); + break; + default: + free(new_seg); + PMIX_ERROR_LOG(PMIX_ERROR); + return NULL; + } + rc = pmix_pshmem.segment_attach(&new_seg->seg_info, PMIX_PSHMEM_RONLY); + if (PMIX_SUCCESS != rc) { + free(new_seg); + new_seg = NULL; + PMIX_ERROR_LOG(rc); + } + return new_seg; +} + +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_extend_segment(pmix_dstore_seg_desc_t *segdesc, const char *base_path, + const char *name, uid_t uid, bool setuid) +{ + pmix_dstore_seg_desc_t *tmp, *seg; + + PMIX_OUTPUT_VERBOSE((2, pmix_gds_base_framework.framework_output, + "%s:%d:%s", + __FILE__, __LINE__, __func__)); + /* find last segment */ + tmp = segdesc; + while (NULL != tmp->next) { + tmp = tmp->next; + } + /* create another segment, the old one is full. */ + seg = pmix_common_dstor_create_new_segment(segdesc->type, base_path, name, tmp->id + 1, uid, setuid); + tmp->next = seg; + + return seg; +} + +PMIX_EXPORT void pmix_common_dstor_delete_sm_desc(pmix_dstore_seg_desc_t *desc) +{ + pmix_dstore_seg_desc_t *tmp; + + /* free all global segments */ + while (NULL != desc) { + tmp = desc->next; + /* detach & unlink from current desc */ + if (desc->seg_info.seg_cpid == getpid()) { + pmix_pshmem.segment_unlink(&desc->seg_info); + } + pmix_pshmem.segment_detach(&desc->seg_info); + free(desc); + desc = tmp; + } +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_segment.h b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_segment.h new file mode 100644 index 0000000000..6430273faa --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/common/dstore/dstore_segment.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" +#include "src/mca/pshmem/base/base.h" + +#include "dstore_common.h" + +#ifndef DSTORE_SEGMENT_H +#define DSTORE_SEGMENT_H + +/* this structs are used to store information about + * shared segments addresses locally at each process, + * so they are common for different types of segments + * and don't have a specific content (namespace's info, + * rank's meta info, ranks's data). */ + +typedef struct pmix_dstore_seg_desc_t pmix_dstore_seg_desc_t; + +typedef enum { + PMIX_DSTORE_INITIAL_SEGMENT, + PMIX_DSTORE_NS_META_SEGMENT, + PMIX_DSTORE_NS_DATA_SEGMENT, + PMIX_DSTORE_NS_LOCK_SEGMENT, +} pmix_dstore_segment_type; + +struct pmix_dstore_seg_desc_t { + pmix_dstore_segment_type type; + pmix_pshmem_seg_t seg_info; + uint32_t id; + pmix_dstore_seg_desc_t *next; +}; + +PMIX_EXPORT int pmix_common_dstor_getpagesize(void); +PMIX_EXPORT size_t pmix_common_dstor_getcacheblocksize(void); +PMIX_EXPORT void pmix_common_dstor_init_segment_info(size_t initial_segment_size, + size_t meta_segment_size, + size_t data_segment_size); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_create_new_segment(pmix_dstore_segment_type type, + const char *base_path, const char *name, uint32_t id, + uid_t uid, bool setuid); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_attach_new_segment(pmix_dstore_segment_type type, + const char *base_path, + const char *name, uint32_t id); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_extend_segment(pmix_dstore_seg_desc_t *segdesc, + const char *base_path, + const char *name, uid_t uid, bool setuid); +PMIX_EXPORT void pmix_common_dstor_delete_sm_desc(pmix_dstore_seg_desc_t *desc); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_create_new_lock_seg(const char *base_path, size_t size, + const char *name, uint32_t id, uid_t uid, bool setuid); +PMIX_EXPORT pmix_dstore_seg_desc_t *pmix_common_dstor_attach_new_lock_seg(const char *base_path, + size_t size, const char *name, uint32_t id); + +#endif // DSTORE_SEGMENT_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/base/base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/base/base.h index 3ada366984..242fc4dabd 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/base/base.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/base/base.h @@ -14,6 +14,7 @@ * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -76,6 +77,12 @@ struct pmix_gds_globals_t { }; typedef struct pmix_gds_globals_t pmix_gds_globals_t; +typedef void * pmix_gds_base_store_modex_cbdata_t; +typedef pmix_status_t (*pmix_gds_base_store_modex_cb_fn_t)(pmix_gds_base_store_modex_cbdata_t cbdata, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_byte_object_t *bo); + PMIX_EXPORT extern pmix_gds_globals_t pmix_gds_globals; /* get a list of available support - caller must free results @@ -98,6 +105,12 @@ PMIX_EXPORT pmix_gds_base_module_t* pmix_gds_base_assign_module(pmix_info_t *inf PMIX_EXPORT pmix_status_t pmix_gds_base_setup_fork(const pmix_proc_t *proc, char ***env); +PMIX_EXPORT pmix_status_t pmix_gds_base_store_modex(struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *xfer, + pmix_gds_base_store_modex_cb_fn_t cb_fn, + pmix_gds_base_store_modex_cbdata_t cbdata); + END_C_DECLS #endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/base/gds_base_fns.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/base/gds_base_fns.c index 16e88485c9..b9f8533c0e 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/base/gds_base_fns.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/base/gds_base_fns.c @@ -3,6 +3,9 @@ * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -83,3 +86,99 @@ pmix_status_t pmix_gds_base_setup_fork(const pmix_proc_t *proc, return PMIX_SUCCESS; } + +pmix_status_t pmix_gds_base_store_modex(struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t * buff, + pmix_gds_base_store_modex_cb_fn_t cb_fn, + pmix_gds_base_store_modex_cbdata_t cbdata) +{ + pmix_status_t rc = PMIX_SUCCESS; + pmix_namespace_t * ns = (pmix_namespace_t *)nspace; + pmix_buffer_t bkt; + pmix_byte_object_t bo, bo2; + int32_t cnt = 1; + char byte; + pmix_collect_t ctype; + bool have_ctype = false; + + /* Loop over the enclosed byte object envelopes and + * store them in our GDS module */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + buff, &bo, &cnt, PMIX_BYTE_OBJECT); + while (PMIX_SUCCESS == rc) { + PMIX_CONSTRUCT(&bkt, pmix_buffer_t); + PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, bo.bytes, bo.size); + /* unpack the data collection flag */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &byte, &cnt, PMIX_BYTE); + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { + /* no data was returned, so we are done with this blob */ + PMIX_DESTRUCT(&bkt); + break; + } + if (PMIX_SUCCESS != rc) { + /* we have an error */ + PMIX_DESTRUCT(&bkt); + goto error; + } + + // Check that this blob was accumulated with the same data collection setting + if (have_ctype) { + if (ctype != (pmix_collect_t)byte) { + rc = PMIX_ERR_INVALID_ARG; + PMIX_DESTRUCT(&bkt); + goto error; + } + } + else { + ctype = (pmix_collect_t)byte; + have_ctype = true; + } + + /* unpack the enclosed blobs from the various peers */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &bo2, &cnt, PMIX_BYTE_OBJECT); + while (PMIX_SUCCESS == rc) { + /* unpack all the kval's from this peer and store them in + * our GDS. Note that PMIx by design holds all data at + * the server level until requested. If our GDS is a + * shared memory region, then the data may be available + * right away - but the client still has to be notified + * of its presence. */ + rc = cb_fn(cbdata, (struct pmix_namespace_t *)ns, cbs, &bo2); + if (PMIX_SUCCESS != rc) { + PMIX_DESTRUCT(&bkt); + goto error; + } + PMIX_BYTE_OBJECT_DESTRUCT(&bo2); + /* get the next blob */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, &bo2, &cnt, PMIX_BYTE_OBJECT); + } + PMIX_DESTRUCT(&bkt); + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { + rc = PMIX_SUCCESS; + } else if (PMIX_SUCCESS != rc) { + goto error; + } + /* unpack and process the next blob */ + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + buff, &bo, &cnt, PMIX_BYTE_OBJECT); + } + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { + rc = PMIX_SUCCESS; + } + +error: + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + + return rc; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/Makefile.am index ac62d8a9aa..dc799c892f 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/Makefile.am @@ -11,7 +11,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2017 Mellanox Technologies, Inc. @@ -24,11 +24,24 @@ # headers = \ - gds_dstore.h + gds_ds12_base.h \ + gds_ds12_lock.h \ + gds_ds12_file.h sources = \ - gds_dstore.c \ - gds_dstore_component.c + gds_ds12_base.c \ + gds_ds12_lock.c \ + gds_ds12_component.c \ + gds_ds12_file.c \ + gds_ds20_file.c + +if HAVE_DSTORE_PTHREAD_LOCK +sources += gds_ds12_lock_pthread.c +else +if HAVE_DSTORE_FCNTL_LOCK +sources += gds_ds12_lock_fcntl.c +endif +endif # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la @@ -49,7 +62,8 @@ endif mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) mca_gds_ds12_la_SOURCES = $(component_sources) -mca_gds_ds12_la_LDFLAGS = -module -avoid-version +mca_gds_ds12_la_LDFLAGS = -module -avoid-version \ + $(PMIX_TOP_BUILDDIR)/src/mca/common/dstore/libmca_common_dstore.la noinst_LTLIBRARIES = $(lib) libmca_gds_ds12_la_SOURCES = $(lib_sources) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_base.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_base.c new file mode 100644 index 0000000000..cdfcb25270 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_base.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include +#include "src/include/pmix_globals.h" +#include "src/util/error.h" +#include "src/mca/gds/base/base.h" +#include "src/util/argv.h" + +#include "src/mca/common/dstore/dstore_common.h" +#include "gds_ds12_base.h" +#include "gds_ds12_lock.h" +#include "gds_ds12_file.h" +#include "src/mca/common/dstore/dstore_base.h" + +static pmix_common_dstore_ctx_t *ds12_ctx; + +static pmix_status_t ds12_init(pmix_info_t info[], size_t ninfo) +{ + pmix_status_t rc = PMIX_SUCCESS; + pmix_common_dstore_file_cbs_t *dstore_file_cbs = NULL; + + if (!PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + dstore_file_cbs = &pmix_ds20_file_module; + } + ds12_ctx = pmix_common_dstor_init("ds12", info, ninfo, + &pmix_ds12_lock_module, + dstore_file_cbs); + if (NULL == ds12_ctx) { + rc = PMIX_ERR_INIT; + } + + return rc; +} + +static void ds12_finalize(void) +{ + pmix_common_dstor_finalize(ds12_ctx); +} + +static pmix_status_t ds12_assign_module(pmix_info_t *info, size_t ninfo, + int *priority) +{ + size_t n, m; + char **options; + + *priority = 20; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { + options = pmix_argv_split(info[n].value.data.string, ','); + for (m=0; NULL != options[m]; m++) { + if (0 == strcmp(options[m], "ds12")) { + /* they specifically asked for us */ + *priority = 100; + break; + } + if (0 == strcmp(options[m], "dstore")) { + /* they are asking for any dstore module - we + * take an intermediate priority in case another + * dstore is more modern than us */ + *priority = 50; + break; + } + } + pmix_argv_free(options); + break; + } + } + } + + return PMIX_SUCCESS; +} + +static pmix_status_t ds12_cache_job_info(struct pmix_namespace_t *ns, + pmix_info_t info[], size_t ninfo) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t ds12_register_job_info(struct pmix_peer_t *pr, + pmix_buffer_t *reply) +{ + if (PMIX_PROC_IS_V1(pr)) { + ds12_ctx->file_cbs = &pmix_ds12_file_module; + } else { + ds12_ctx->file_cbs = &pmix_ds20_file_module; + } + return pmix_common_dstor_register_job_info(ds12_ctx, pr, reply); +} + +static pmix_status_t ds12_store_job_info(const char *nspace, pmix_buffer_t *buf) +{ + return pmix_common_dstor_store_job_info(ds12_ctx, nspace, buf); +} + +static pmix_status_t ds12_store(const pmix_proc_t *proc, + pmix_scope_t scope, + pmix_kval_t *kv) +{ + return pmix_common_dstor_store(ds12_ctx, proc, scope, kv); +} + +/* this function is only called by the PMIx server when its + * host has received data from some other peer. It therefore + * always contains data solely from remote procs, and we + * shall store it accordingly */ +static pmix_status_t ds12_store_modex(struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *buf) +{ + return pmix_common_dstor_store_modex(ds12_ctx, nspace, cbs, buf); +} + +static pmix_status_t ds12_fetch(const pmix_proc_t *proc, + pmix_scope_t scope, bool copy, + const char *key, + pmix_info_t info[], size_t ninfo, + pmix_list_t *kvs) +{ + return pmix_common_dstor_fetch(ds12_ctx, proc, scope, copy, key, info, ninfo, kvs); +} + +static pmix_status_t ds12_setup_fork(const pmix_proc_t *peer, char ***env) +{ + return pmix_common_dstor_setup_fork(ds12_ctx, PMIX_DSTORE_ESH_BASE_PATH, peer, env); +} + +static pmix_status_t ds12_add_nspace(const char *nspace, + pmix_info_t info[], + size_t ninfo) +{ + return pmix_common_dstor_add_nspace(ds12_ctx, nspace, info, ninfo); +} + +static pmix_status_t ds12_del_nspace(const char* nspace) +{ + return pmix_common_dstor_del_nspace(ds12_ctx, nspace); +} + +pmix_gds_base_module_t pmix_ds12_module = { + .name = "ds12", + .is_tsafe = false, + .init = ds12_init, + .finalize = ds12_finalize, + .assign_module = ds12_assign_module, + .cache_job_info = ds12_cache_job_info, + .register_job_info = ds12_register_job_info, + .store_job_info = ds12_store_job_info, + .store = ds12_store, + .store_modex = ds12_store_modex, + .fetch = ds12_fetch, + .setup_fork = ds12_setup_fork, + .add_nspace = ds12_add_nspace, + .del_nspace = ds12_del_nspace, +}; + diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_base.h new file mode 100644 index 0000000000..d208e2d154 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_base.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef GDS_DSTORE_12_H +#define GDS_DSTORE_12_H + +#include "src/mca/gds/gds.h" + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_gds_base_component_t mca_gds_ds12_component; +extern pmix_gds_base_module_t pmix_ds12_module; + +#endif // GDS_DSTORE_12_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_component.c similarity index 99% rename from opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore_component.c rename to opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_component.c index 35d984e1b9..9f52d4fe99 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore_component.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_component.c @@ -33,7 +33,7 @@ #include "src/include/pmix_globals.h" #include "src/mca/gds/gds.h" -#include "gds_dstore.h" +#include "gds_ds12_base.h" static pmix_status_t component_open(void); static pmix_status_t component_close(void); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_file.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_file.c new file mode 100644 index 0000000000..701578d83a --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_file.c @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" + +#include "src/mca/common/dstore/dstore_file.h" +#include "gds_ds12_file.h" + +#define ESH_KEY_SIZE_V12(key, size) \ +__pmix_attribute_extension__ ({ \ + size_t len = strlen((char*)key) + 1 + sizeof(size_t) + size; \ + len; \ +}) + +/* in ext slot new offset will be stored in case if + * new data were added for the same process during + * next commit + */ +#define EXT_SLOT_SIZE_V12() \ + (ESH_KEY_SIZE_V12(ESH_REGION_EXTENSION, sizeof(size_t))) + +#define ESH_KV_SIZE_V12(addr) \ +__pmix_attribute_extension__ ({ \ + size_t sz; \ + memcpy(&sz, addr + \ + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), \ + sizeof(size_t)); \ + sz += ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)) + \ + sizeof(size_t); \ + sz; \ +}) + +#define ESH_KNAME_PTR_V12(addr) \ +__pmix_attribute_extension__ ({ \ + char *name_ptr = (char*)addr; \ + name_ptr; \ +}) + +#define ESH_KNAME_LEN_V12(key) \ +__pmix_attribute_extension__ ({ \ + size_t len = strlen((char*)key) + 1; \ + len; \ +}) + +#define ESH_DATA_PTR_V12(addr) \ +__pmix_attribute_extension__ ({ \ + uint8_t *data_ptr = \ + addr + \ + sizeof(size_t) + \ + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)); \ + data_ptr; \ +}) + +#define ESH_DATA_SIZE_V12(addr) \ +__pmix_attribute_extension__ ({ \ + size_t data_size; \ + memcpy(&data_size, \ + addr + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), \ + sizeof(size_t)); \ + data_size; \ +}) + +#define ESH_PUT_KEY_V12(addr, key, buffer, size) \ +__pmix_attribute_extension__ ({ \ + size_t sz = size; \ + memset(addr, 0, ESH_KNAME_LEN_V12(key)); \ + strncpy((char *)addr, key, ESH_KNAME_LEN_V12(key)); \ + memcpy(addr + ESH_KNAME_LEN_V12(key), &sz, \ + sizeof(size_t)); \ + memcpy(addr + ESH_KNAME_LEN_V12(key) + sizeof(size_t), \ + buffer, size); \ +}) + +static size_t pmix_ds12_kv_size(uint8_t *addr) +{ + size_t size; + + memcpy(&size, addr + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)), + sizeof(size_t)); + size += ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)) + sizeof(size_t); + return size; +} + +static char* pmix_ds12_key_name_ptr(uint8_t *addr) +{ + return ESH_KNAME_PTR_V12(addr); +} + +static size_t pmix_ds12_key_name_len(char *key) +{ + return ESH_KNAME_LEN_V12(key); +} + +static uint8_t* pmix_ds12_data_ptr(uint8_t *addr) +{ + return ESH_DATA_PTR_V12(addr); +} + +static size_t pmix_ds12_data_size(uint8_t *addr, uint8_t* data_ptr) +{ + return ESH_DATA_SIZE_V12(addr); +} + +static size_t pmix_ds12_key_size(char *addr, size_t data_size) +{ + return ESH_KEY_SIZE_V12(addr, data_size); +} + +static size_t pmix_ds12_ext_slot_size(void) +{ + return EXT_SLOT_SIZE_V12(); +} + +static int pmix_ds12_put_key(uint8_t *addr, char *key, void *buf, size_t size) +{ + ESH_PUT_KEY_V12(addr, key, buf, size); + return PMIX_SUCCESS; +} + +static bool pmix_ds12_is_invalid(uint8_t *addr) +{ + bool ret = (0 == strncmp(ESH_REGION_INVALIDATED, ESH_KNAME_PTR_V12(addr), + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)))); + return ret; +} + +static void pmix_ds12_set_invalid(uint8_t *addr) +{ + strncpy(ESH_KNAME_PTR_V12(addr), ESH_REGION_INVALIDATED, + ESH_KNAME_LEN_V12(ESH_REGION_INVALIDATED)); +} + +static bool pmix_ds12_is_ext_slot(uint8_t *addr) +{ + bool ret; + ret = (0 == strncmp(ESH_REGION_EXTENSION, ESH_KNAME_PTR_V12(addr), + ESH_KNAME_LEN_V12(ESH_KNAME_PTR_V12(addr)))); + return ret; +} + +static bool pmix_ds12_kname_match(uint8_t *addr, const char *key, size_t key_hash) +{ + bool ret = 0; + + ret = (0 == strncmp(ESH_KNAME_PTR_V12(addr), + key, ESH_KNAME_LEN_V12(key))); + return ret; +} + +pmix_common_dstore_file_cbs_t pmix_ds12_file_module = { + .name = "ds12", + .kval_size = pmix_ds12_kv_size, + .kname_ptr = pmix_ds12_key_name_ptr, + .kname_len = pmix_ds12_key_name_len, + .data_ptr = pmix_ds12_data_ptr, + .data_size = pmix_ds12_data_size, + .key_size = pmix_ds12_key_size, + .ext_slot_size = pmix_ds12_ext_slot_size, + .put_key = pmix_ds12_put_key, + .is_invalid = pmix_ds12_is_invalid, + .is_extslot = pmix_ds12_is_ext_slot, + .set_invalid = pmix_ds12_set_invalid, + .key_hash = NULL, + .key_match = pmix_ds12_kname_match +}; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_file.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_file.h new file mode 100644 index 0000000000..33c8864743 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_file.h @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef GDS_DS12_FILE_H +#define GDS_DS12_FILE_H + +#include +#include + +extern pmix_common_dstore_file_cbs_t pmix_ds12_file_module; +extern pmix_common_dstore_file_cbs_t pmix_ds20_file_module; + +#endif // GDS_DS12_FILE_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock.c new file mode 100644 index 0000000000..9872dd7c4d --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include "src/mca/common/dstore/dstore_common.h" + +#include "gds_ds12_lock.h" + +pmix_common_lock_callbacks_t pmix_ds12_lock_module = { + .init = pmix_gds_ds12_lock_init, + .finalize = pmix_ds12_lock_finalize, + .rd_lock = pmix_ds12_lock_rd_get, + .rd_unlock = pmix_ds12_lock_rw_rel, + .wr_lock = pmix_ds12_lock_wr_get, + .wr_unlock = pmix_ds12_lock_rw_rel +}; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock.h new file mode 100644 index 0000000000..9d9b91ad1e --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef DS12_LOCK_H +#define DS12_LOCK_H + +#include +#include + +#include "src/mca/common/dstore/dstore_common.h" + +pmix_status_t pmix_gds_ds12_lock_init(pmix_common_dstor_lock_ctx_t *lock_ctx, + const char *base_path, const char *name, + uint32_t local_size, uid_t uid, bool setuid); +void pmix_ds12_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx); +pmix_status_t pmix_ds12_lock_rd_get(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds12_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds12_lock_rw_rel(pmix_common_dstor_lock_ctx_t lock_ctx); + +extern pmix_common_lock_callbacks_t pmix_ds12_lock_module; + +#endif // DS12_LOCK_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock_fcntl.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock_fcntl.c new file mode 100644 index 0000000000..477e91465f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock_fcntl.c @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif + +#include + +#include "src/mca/common/dstore/dstore_common.h" +#include "src/mca/gds/base/base.h" + +#include "src/util/error.h" +#include "src/util/output.h" + +#include "gds_ds12_lock.h" + +#define _ESH_12_FCNTL_LOCK(lockfd, operation) \ +__pmix_attribute_extension__ ({ \ + pmix_status_t ret = PMIX_SUCCESS; \ + int i; \ + struct flock fl = {0}; \ + fl.l_type = operation; \ + fl.l_whence = SEEK_SET; \ + for(i = 0; i < 10; i++) { \ + if( 0 > fcntl(lockfd, F_SETLKW, &fl) ) { \ + switch( errno ){ \ + case EINTR: \ + continue; \ + case ENOENT: \ + case EINVAL: \ + ret = PMIX_ERR_NOT_FOUND; \ + break; \ + case EBADF: \ + ret = PMIX_ERR_BAD_PARAM; \ + break; \ + case EDEADLK: \ + case EFAULT: \ + case ENOLCK: \ + ret = PMIX_ERR_RESOURCE_BUSY; \ + break; \ + default: \ + ret = PMIX_ERROR; \ + break; \ + } \ + } \ + break; \ + } \ + if (ret) { \ + pmix_output(0, "%s %d:%s lock failed: %s", \ + __FILE__, __LINE__, __func__, strerror(errno)); \ + } \ + ret; \ +}) + +typedef struct { + char *lockfile; + int lockfd; +} ds12_lock_fcntl_ctx_t; + +pmix_status_t pmix_gds_ds12_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, + const char *name, uint32_t local_size, uid_t uid, bool setuid) +{ + pmix_status_t rc = PMIX_SUCCESS; + ds12_lock_fcntl_ctx_t *lock_ctx; + + if (*ctx != NULL) { + return PMIX_SUCCESS; + } + + lock_ctx = (ds12_lock_fcntl_ctx_t*)malloc(sizeof(ds12_lock_fcntl_ctx_t)); + if (NULL == lock_ctx) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + *ctx = lock_ctx; + memset(lock_ctx, 0, sizeof(ds12_lock_fcntl_ctx_t)); + lock_ctx->lockfd = -1; + + /* create a lock file to prevent clients from reading while server is writing + * to the shared memory. This situation is quite often, especially in case of + * direct modex when clients might ask for data simultaneously. */ + if(0 > asprintf(&lock_ctx->lockfile, "%s/dstore_sm.lock", base_path)) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + PMIX_ERROR_LOG(rc); + goto error; + } + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, lock_ctx->lockfile)); + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + lock_ctx->lockfd = open(lock_ctx->lockfile, O_CREAT | O_RDWR | O_EXCL, 0600); + + /* if previous launch was crashed, the lockfile might not be deleted and unlocked, + * so we delete it and create a new one. */ + if (lock_ctx->lockfd < 0) { + unlink(lock_ctx->lockfile); + lock_ctx->lockfd = open(lock_ctx->lockfile, O_CREAT | O_RDWR, 0600); + if (lock_ctx->lockfd < 0) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + } + if (0 != setuid) { + if (0 > chown(lock_ctx->lockfile, uid, (gid_t) -1)) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + if (0 > chmod(lock_ctx->lockfile, S_IRUSR | S_IWGRP | S_IRGRP)) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + } + } + else { + lock_ctx->lockfd = open(lock_ctx->lockfile, O_RDONLY); + if (0 > lock_ctx->lockfd) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + } + + return rc; + +error: + if (NULL != lock_ctx) { + if (NULL != lock_ctx->lockfile) { + free(lock_ctx->lockfile); + } + if (0 > lock_ctx->lockfd) { + close(lock_ctx->lockfd); + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + unlink(lock_ctx->lockfile); + } + } + free(lock_ctx); + lock_ctx = NULL; + } + *ctx = NULL; + + return rc; +} + +void pmix_ds12_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx) +{ + ds12_lock_fcntl_ctx_t *fcntl_lock = (ds12_lock_fcntl_ctx_t*)*lock_ctx; + + if (NULL == fcntl_lock) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); + return; + } + + close(fcntl_lock->lockfd); + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + unlink(fcntl_lock->lockfile); + } + free(fcntl_lock); + *lock_ctx = NULL; +} + +pmix_status_t pmix_ds12_lock_rd_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ ds12_lock_fcntl_ctx_t *fcntl_lock = (ds12_lock_fcntl_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == fcntl_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_FCNTL_LOCK(fcntl_lock->lockfd, F_RDLCK); + + return rc; + +} + +pmix_status_t pmix_ds12_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ ds12_lock_fcntl_ctx_t *fcntl_lock = (ds12_lock_fcntl_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == fcntl_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_FCNTL_LOCK(fcntl_lock->lockfd, F_WRLCK); + + return rc; + +} + +pmix_status_t pmix_ds12_lock_rw_rel(pmix_common_dstor_lock_ctx_t lock_ctx) +{ ds12_lock_fcntl_ctx_t *fcntl_lock = (ds12_lock_fcntl_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == fcntl_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_FCNTL_LOCK(fcntl_lock->lockfd, F_UNLCK); + + return rc; + +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock_pthread.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock_pthread.c new file mode 100644 index 0000000000..163015856e --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds12_lock_pthread.c @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#include + +#include "src/mca/common/dstore/dstore_common.h" +#include "src/mca/gds/base/base.h" +#include "src/mca/pshmem/pshmem.h" + +#include "src/util/error.h" +#include "src/util/output.h" + +#include "gds_ds12_lock.h" +#include "src/mca/common/dstore/dstore_segment.h" + +#define _ESH_12_PTHREAD_LOCK(rwlock, func) \ +__pmix_attribute_extension__ ({ \ + pmix_status_t ret = PMIX_SUCCESS; \ + int rc; \ + rc = pthread_rwlock_##func(rwlock); \ + if (0 != rc) { \ + switch (errno) { \ + case EINVAL: \ + ret = PMIX_ERR_INIT; \ + break; \ + case EPERM: \ + ret = PMIX_ERR_NO_PERMISSIONS; \ + break; \ + } \ + } \ + if (ret) { \ + pmix_output(0, "%s %d:%s lock failed: %s", \ + __FILE__, __LINE__, __func__, strerror(errno)); \ + } \ + ret; \ +}) + +typedef struct { + char *lockfile; + pmix_pshmem_seg_t *segment; + pthread_rwlock_t *rwlock; +} ds12_lock_pthread_ctx_t; + +pmix_status_t pmix_gds_ds12_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, + const char * name, uint32_t local_size, uid_t uid, bool setuid) +{ + size_t size = pmix_common_dstor_getpagesize(); + pmix_status_t rc = PMIX_SUCCESS; + pthread_rwlockattr_t attr; + ds12_lock_pthread_ctx_t *lock_ctx = (ds12_lock_pthread_ctx_t*)ctx; + + if (*ctx != NULL) { + return PMIX_SUCCESS; + } + + lock_ctx = (ds12_lock_pthread_ctx_t*)malloc(sizeof(ds12_lock_pthread_ctx_t)); + if (NULL == lock_ctx) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + memset(lock_ctx, 0, sizeof(ds12_lock_pthread_ctx_t)); + *ctx = (pmix_common_dstor_lock_ctx_t*)lock_ctx; + + lock_ctx->segment = (pmix_pshmem_seg_t *)malloc(sizeof(pmix_pshmem_seg_t)); + if (NULL == lock_ctx->segment) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + PMIX_ERROR_LOG(rc); + goto error; + } + + /* create a lock file to prevent clients from reading while server is writing + * to the shared memory. This situation is quite often, especially in case of + * direct modex when clients might ask for data simultaneously. */ + if(0 > asprintf(&lock_ctx->lockfile, "%s/dstore_sm.lock", base_path)) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + PMIX_ERROR_LOG(rc); + goto error; + } + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s _lockfile_name: %s", __FILE__, __LINE__, __func__, lock_ctx->lockfile)); + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_create(lock_ctx->segment, + lock_ctx->lockfile, size))) { + PMIX_ERROR_LOG(rc); + goto error; + } + memset(lock_ctx->segment->seg_base_addr, 0, size); + if (0 != setuid) { + if (0 > chown(lock_ctx->lockfile, (uid_t) uid, (gid_t) -1)){ + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + /* set the mode as required */ + if (0 > chmod(lock_ctx->lockfile, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP )) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + } + lock_ctx->rwlock = (pthread_rwlock_t *)lock_ctx->segment->seg_base_addr; + + if (0 != pthread_rwlockattr_init(&attr)) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + goto error; + } + if (0 != pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { + pthread_rwlockattr_destroy(&attr); + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } +#ifdef HAVE_PTHREAD_SETKIND + if (0 != pthread_rwlockattr_setkind_np(&attr, + PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)) { + pthread_rwlockattr_destroy(&attr); + PMIX_ERROR_LOG(PMIX_ERR_INIT); + goto error; + } +#endif + if (0 != pthread_rwlock_init(lock_ctx->rwlock, &attr)) { + pthread_rwlockattr_destroy(&attr); + PMIX_ERROR_LOG(PMIX_ERR_INIT); + goto error; + } + if (0 != pthread_rwlockattr_destroy(&attr)) { + PMIX_ERROR_LOG(PMIX_ERR_INIT); + goto error; + } + + } + else { + lock_ctx->segment->seg_size = size; + snprintf(lock_ctx->segment->seg_name, PMIX_PATH_MAX, "%s", lock_ctx->lockfile); + if (PMIX_SUCCESS != (rc = pmix_pshmem.segment_attach(lock_ctx->segment, + PMIX_PSHMEM_RW))) { + PMIX_ERROR_LOG(rc); + goto error; + } + lock_ctx->rwlock = (pthread_rwlock_t *)lock_ctx->segment->seg_base_addr; + } + + return PMIX_SUCCESS; + +error: + if (NULL != lock_ctx) { + if (lock_ctx->segment) { + /* detach & unlink from current desc */ + if (lock_ctx->segment->seg_cpid == getpid()) { + pmix_pshmem.segment_unlink(lock_ctx->segment); + } + pmix_pshmem.segment_detach(lock_ctx->segment); + lock_ctx->rwlock = NULL; + } + if (NULL != lock_ctx->lockfile) { + free(lock_ctx->lockfile); + } + free(lock_ctx); + *ctx = (pmix_common_dstor_lock_ctx_t*)NULL; + } + + return rc; +} + +void pmix_ds12_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx) +{ + ds12_lock_pthread_ctx_t *pthread_lock = + (ds12_lock_pthread_ctx_t*)*lock_ctx; + + if (NULL == pthread_lock) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); + return; + } + if (0 != pthread_rwlock_destroy(pthread_lock->rwlock)) { + PMIX_ERROR_LOG(PMIX_ERROR); + return; + } + + if (NULL == pthread_lock->segment) { + PMIX_ERROR_LOG(PMIX_ERROR); + return; + } + if (NULL == pthread_lock->lockfile) { + PMIX_ERROR_LOG(PMIX_ERROR); + return; + } + + /* detach & unlink from current desc */ + if (pthread_lock->segment->seg_cpid == getpid()) { + pmix_pshmem.segment_unlink(pthread_lock->segment); + } + pmix_pshmem.segment_detach(pthread_lock->segment); + + free(pthread_lock->segment); + pthread_lock->segment = NULL; + free(pthread_lock->lockfile); + pthread_lock->lockfile = NULL; + pthread_lock->rwlock = NULL; + free(pthread_lock); + *lock_ctx = NULL; +} + +pmix_status_t pmix_ds12_lock_rd_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + ds12_lock_pthread_ctx_t *pthread_lock = (ds12_lock_pthread_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == pthread_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_PTHREAD_LOCK(pthread_lock->rwlock, rdlock); + + return rc; +} + +pmix_status_t pmix_ds12_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + ds12_lock_pthread_ctx_t *pthread_lock = (ds12_lock_pthread_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == pthread_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_PTHREAD_LOCK(pthread_lock->rwlock, wrlock); + + return rc; +} + +pmix_status_t pmix_ds12_lock_rw_rel(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + ds12_lock_pthread_ctx_t *pthread_lock = (ds12_lock_pthread_ctx_t*)lock_ctx; + pmix_status_t rc; + + if (NULL == pthread_lock) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = _ESH_12_PTHREAD_LOCK(pthread_lock->rwlock, unlock); + + return rc; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds20_file.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds20_file.c new file mode 100644 index 0000000000..d50cb8124c --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_ds20_file.c @@ -0,0 +1,166 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" + +#include "src/mca/common/dstore/dstore_file.h" +#include "gds_ds12_file.h" + +#define ESH_KV_SIZE_V20(addr) \ +__pmix_attribute_extension__ ({ \ + size_t sz; \ + memcpy(&sz, addr, sizeof(size_t)); \ + sz; \ +}) + +#define ESH_KNAME_PTR_V20(addr) \ + ((char *)addr + sizeof(size_t)) + +#define ESH_KNAME_LEN_V20(key) \ +__pmix_attribute_extension__ ({ \ + size_t kname_len = strlen(key) + 1; \ + size_t len = (kname_len < ESH_MIN_KEY_LEN) ? \ + ESH_MIN_KEY_LEN : kname_len; \ + len; \ +}) + +#define ESH_DATA_PTR_V20(addr) \ +__pmix_attribute_extension__ ({ \ + size_t kname_len = \ + ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr)); \ + uint8_t *data_ptr = addr + sizeof(size_t) + kname_len; \ + data_ptr; \ +}) + +#define ESH_DATA_SIZE_V20(addr, data_ptr) \ +__pmix_attribute_extension__ ({ \ + size_t __sz = ESH_KV_SIZE_V20(addr); \ + size_t data_size = __sz - (data_ptr - addr); \ + data_size; \ +}) + +#define ESH_KEY_SIZE_V20(key, size) \ + (sizeof(size_t) + ESH_KNAME_LEN_V20((char*)key) + size) + +/* in ext slot new offset will be stored in case if + * new data were added for the same process during + * next commit + */ +#define EXT_SLOT_SIZE_V20() \ + (ESH_KEY_SIZE_V20(ESH_REGION_EXTENSION, sizeof(size_t))) + + +#define ESH_PUT_KEY_V20(addr, key, buffer, size) \ +__pmix_attribute_extension__ ({ \ + size_t sz = ESH_KEY_SIZE_V20(key, size); \ + memcpy(addr, &sz, sizeof(size_t)); \ + memset(addr + sizeof(size_t), 0, \ + ESH_KNAME_LEN_V20(key)); \ + strncpy((char *)addr + sizeof(size_t), \ + key, ESH_KNAME_LEN_V20(key)); \ + memcpy(addr + sizeof(size_t) + ESH_KNAME_LEN_V20(key), \ + buffer, size); \ +}) + +static size_t pmix_ds20_kv_size(uint8_t *key) +{ + size_t size; + + memcpy(&size, key, sizeof(size_t)); + return size; +} + +static char* pmix_ds20_key_name_ptr(uint8_t *addr) +{ + return ESH_KNAME_PTR_V20(addr); +} + +static size_t pmix_ds20_key_name_len(char *key) +{ + return ESH_KNAME_LEN_V20(key); +} + +static uint8_t* pmix_ds20_data_ptr(uint8_t *addr) +{ + return ESH_DATA_PTR_V20(addr); +} + +static size_t pmix_ds20_data_size(uint8_t *addr, uint8_t* data_ptr) +{ + return ESH_DATA_SIZE_V20(addr, data_ptr); +} + +static size_t pmix_ds20_key_size(char *addr, size_t data_size) +{ + return ESH_KEY_SIZE_V20(addr, data_size); +} + +static size_t pmix_ds20_ext_slot_size(void) +{ + return EXT_SLOT_SIZE_V20(); +} + +static int pmix_ds20_put_key(uint8_t *addr, char *key, void *buf, size_t size) +{ + ESH_PUT_KEY_V20(addr, key, buf, size); + return PMIX_SUCCESS; +} + +static bool pmix_ds20_is_invalid(uint8_t *addr) +{ + bool ret = (0 == strncmp(ESH_REGION_INVALIDATED, ESH_KNAME_PTR_V20(addr), + ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr)))); + return ret; +} + +static void pmix_ds20_set_invalid(uint8_t *addr) +{ + strncpy(ESH_KNAME_PTR_V20(addr), ESH_REGION_INVALIDATED, + ESH_KNAME_LEN_V20(ESH_REGION_INVALIDATED)); +} + +static bool pmix_ds20_is_ext_slot(uint8_t *addr) +{ + bool ret; + ret = (0 == strncmp(ESH_REGION_EXTENSION, ESH_KNAME_PTR_V20(addr), + ESH_KNAME_LEN_V20(ESH_KNAME_PTR_V20(addr)))); + return ret; +} + +static bool pmix_ds20_kname_match(uint8_t *addr, const char *key, size_t key_hash) +{ + bool ret = 0; + + ret = (0 == strncmp(ESH_KNAME_PTR_V20(addr), + key, ESH_KNAME_LEN_V20(key))); + return ret; +} + + +pmix_common_dstore_file_cbs_t pmix_ds20_file_module = { + .name = "ds20", + .kval_size = pmix_ds20_kv_size, + .kname_ptr = pmix_ds20_key_name_ptr, + .kname_len = pmix_ds20_key_name_len, + .data_ptr = pmix_ds20_data_ptr, + .data_size = pmix_ds20_data_size, + .key_size = pmix_ds20_key_size, + .ext_slot_size = pmix_ds20_ext_slot_size, + .put_key = pmix_ds20_put_key, + .is_invalid = pmix_ds20_is_invalid, + .is_extslot = pmix_ds20_is_ext_slot, + .set_invalid = pmix_ds20_set_invalid, + .key_hash = NULL, + .key_match = pmix_ds20_kname_match +}; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.h deleted file mode 100644 index abd4723ad2..0000000000 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds12/gds_dstore.h +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2017 Mellanox Technologies, Inc. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef PMIX_DS12_H -#define PMIX_DS12_H - -#include - - -#include "src/mca/gds/gds.h" -#include "src/mca/pshmem/pshmem.h" - -BEGIN_C_DECLS - -#include -#include "src/class/pmix_value_array.h" - -#define INITIAL_SEG_SIZE 4096 -#define NS_META_SEG_SIZE (1<<22) -#define NS_DATA_SEG_SIZE (1<<22) - -#define PMIX_DSTORE_ESH_BASE_PATH "PMIX_DSTORE_ESH_BASE_PATH" - -#ifdef HAVE_PTHREAD_SHARED -#define ESH_PTHREAD_LOCK -#elif defined HAVE_FCNTL_FLOCK -#define ESH_FCNTL_LOCK -#else -#error No locking mechanism was found -#endif - -/* this structs are used to store information about - * shared segments addresses locally at each process, - * so they are common for different types of segments - * and don't have a specific content (namespace's info, - * rank's meta info, ranks's data). */ - -typedef enum { - INITIAL_SEGMENT, - NS_META_SEGMENT, - NS_DATA_SEGMENT -} segment_type; - -typedef struct seg_desc_t seg_desc_t; -struct seg_desc_t { - segment_type type; - pmix_pshmem_seg_t seg_info; - uint32_t id; - seg_desc_t *next; -}; - -typedef struct ns_map_data_s ns_map_data_t; -typedef struct session_s session_t; -typedef struct ns_map_s ns_map_t; - -struct session_s { - int in_use; - uid_t jobuid; - char setjobuid; - char *nspace_path; - char *lockfile; -#ifdef ESH_PTHREAD_LOCK - pmix_pshmem_seg_t *rwlock_seg; - pthread_rwlock_t *rwlock; -#endif - int lockfd; - seg_desc_t *sm_seg_first; - seg_desc_t *sm_seg_last; -}; - -struct ns_map_data_s { - char name[PMIX_MAX_NSLEN+1]; - size_t tbl_idx; - int track_idx; -}; - -struct ns_map_s { - int in_use; - ns_map_data_t data; -}; - -/* initial segment format: - * size_t num_elems; - * size_t full; //indicate to client that it needs to attach to the next segment - * ns_seg_info_t ns_seg_info[max_ns_num]; - */ - -typedef struct { - ns_map_data_t ns_map; - size_t num_meta_seg;/* read by clients to attach to this number of segments. */ - size_t num_data_seg; -} ns_seg_info_t; - -/* meta segment format: - * size_t num_elems; - * rank_meta_info meta_info[max_meta_elems]; - */ - -typedef struct { - size_t rank; - size_t offset; - size_t count; -} rank_meta_info; - -typedef struct { - pmix_value_array_t super; - ns_map_data_t ns_map; - size_t num_meta_seg; - size_t num_data_seg; - seg_desc_t *meta_seg; - seg_desc_t *data_seg; - bool in_use; -} ns_track_elem_t; - -/* the component must be visible data for the linker to find it */ -PMIX_EXPORT extern pmix_gds_base_component_t mca_gds_ds12_component; -extern pmix_gds_base_module_t pmix_ds12_module; - -END_C_DECLS - -#endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/Makefile.am new file mode 100644 index 0000000000..215275754d --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/Makefile.am @@ -0,0 +1,62 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2017 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# Copyright (c) 2017-2018 Mellanox Technologies, Inc. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers = \ + gds_ds21_base.h \ + gds_ds21_lock.h \ + gds_ds21_file.h + +sources = \ + gds_ds21_base.c \ + gds_ds21_lock.c \ + gds_ds21_lock_pthread.c \ + gds_ds21_component.c \ + gds_ds21_file.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_gds_ds21_DSO +lib = +lib_sources = +component = mca_gds_ds21.la +component_sources = $(headers) $(sources) +else +lib = libmca_gds_ds21.la +lib_sources = $(headers) $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component) +mca_gds_ds21_la_SOURCES = $(component_sources) +mca_gds_ds21_la_LDFLAGS = -module -avoid-version \ + $(PMIX_TOP_BUILDDIR)/src/mca/common/dstore/libmca_common_dstore.la + +noinst_LTLIBRARIES = $(lib) +libmca_gds_ds21_la_SOURCES = $(lib_sources) +libmca_gds_ds21_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_base.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_base.c new file mode 100644 index 0000000000..a3f32c9c26 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_base.c @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include +#include "src/include/pmix_globals.h" +#include "src/util/error.h" +#include "src/mca/gds/base/base.h" +#include "src/util/argv.h" + +#include "src/mca/common/dstore/dstore_common.h" +#include "gds_ds21_base.h" +#include "gds_ds21_lock.h" +#include "gds_ds21_file.h" +#include "src/mca/common/dstore/dstore_base.h" + +static pmix_common_dstore_ctx_t *ds21_ctx; + +static pmix_status_t ds21_init(pmix_info_t info[], size_t ninfo) +{ + pmix_status_t rc = PMIX_SUCCESS; + + ds21_ctx = pmix_common_dstor_init("ds21", info, ninfo, + &pmix_ds21_lock_module, + &pmix_ds21_file_module); + if (NULL == ds21_ctx) { + rc = PMIX_ERR_INIT; + } + + return rc; +} + +static void ds21_finalize(void) +{ + pmix_common_dstor_finalize(ds21_ctx); +} + +static pmix_status_t ds21_assign_module(pmix_info_t *info, size_t ninfo, + int *priority) +{ + size_t n, m; + char **options; + + *priority = 20; + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_GDS_MODULE, PMIX_MAX_KEYLEN)) { + options = pmix_argv_split(info[n].value.data.string, ','); + for (m=0; NULL != options[m]; m++) { + if (0 == strcmp(options[m], "ds21")) { + /* they specifically asked for us */ + *priority = 120; + break; + } + if (0 == strcmp(options[m], "dstore")) { + *priority = 60; + break; + } + } + pmix_argv_free(options); + break; + } + } + } + + return PMIX_SUCCESS; +} + +static pmix_status_t ds21_cache_job_info(struct pmix_namespace_t *ns, + pmix_info_t info[], size_t ninfo) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t ds21_register_job_info(struct pmix_peer_t *pr, + pmix_buffer_t *reply) +{ + return pmix_common_dstor_register_job_info(ds21_ctx, pr, reply); +} + +static pmix_status_t ds21_store_job_info(const char *nspace, pmix_buffer_t *buf) +{ + return pmix_common_dstor_store_job_info(ds21_ctx, nspace, buf); +} + +static pmix_status_t ds21_store(const pmix_proc_t *proc, + pmix_scope_t scope, + pmix_kval_t *kv) +{ + return pmix_common_dstor_store(ds21_ctx, proc, scope, kv); +} + +/* this function is only called by the PMIx server when its + * host has received data from some other peer. It therefore + * always contains data solely from remote procs, and we + * shall store it accordingly */ +static pmix_status_t ds21_store_modex(struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_buffer_t *buf) +{ + return pmix_common_dstor_store_modex(ds21_ctx, nspace, cbs, buf); +} + +static pmix_status_t ds21_fetch(const pmix_proc_t *proc, + pmix_scope_t scope, bool copy, + const char *key, + pmix_info_t info[], size_t ninfo, + pmix_list_t *kvs) +{ + return pmix_common_dstor_fetch(ds21_ctx, proc, scope, copy, key, info, ninfo, kvs); +} + +static pmix_status_t ds21_setup_fork(const pmix_proc_t *peer, char ***env) +{ + pmix_status_t rc; + char *env_name = NULL; + int ds_ver = 0; + + sscanf(ds21_ctx->ds_name, "ds%d", &ds_ver); + if (0 == ds_ver) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + return rc; + } + if (0 > asprintf(&env_name, PMIX_DSTORE_VER_BASE_PATH_FMT, ds_ver)) { + rc = PMIX_ERR_NOMEM; + PMIX_ERROR_LOG(rc); + return rc; + } + rc = pmix_common_dstor_setup_fork(ds21_ctx, env_name, peer, env); + free(env_name); + + return rc; +} + +static pmix_status_t ds21_add_nspace(const char *nspace, + pmix_info_t info[], + size_t ninfo) +{ + return pmix_common_dstor_add_nspace(ds21_ctx, nspace, info, ninfo); +} + +static pmix_status_t ds21_del_nspace(const char* nspace) +{ + return pmix_common_dstor_del_nspace(ds21_ctx, nspace); +} + +pmix_gds_base_module_t pmix_ds21_module = { + .name = "ds21", + .is_tsafe = true, + .init = ds21_init, + .finalize = ds21_finalize, + .assign_module = ds21_assign_module, + .cache_job_info = ds21_cache_job_info, + .register_job_info = ds21_register_job_info, + .store_job_info = ds21_store_job_info, + .store = ds21_store, + .store_modex = ds21_store_modex, + .fetch = ds21_fetch, + .setup_fork = ds21_setup_fork, + .add_nspace = ds21_add_nspace, + .del_nspace = ds21_del_nspace, +}; + diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_base.h new file mode 100644 index 0000000000..c8fc4d43e5 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_base.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef GDS_DSTORE_21_H +#define GDS_DSTORE_21_H + +#include "src/mca/gds/gds.h" + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_gds_base_component_t mca_gds_ds21_component; +extern pmix_gds_base_module_t pmix_ds21_module; + +#endif // GDS_DSTORE_21_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_component.c new file mode 100644 index 0000000000..c1f42944df --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_component.c @@ -0,0 +1,93 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include +#include "pmix_common.h" + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/gds.h" +#include "gds_ds21_base.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +pmix_gds_base_component_t mca_gds_ds21_component = { + .base = { + PMIX_GDS_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "ds21", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + } +}; + + +static int component_open(void) +{ + return PMIX_SUCCESS; +} + + +static int component_query(pmix_mca_base_module_t **module, int *priority) +{ + /* launchers cannot use the dstore */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + *priority = 0; + *module = NULL; + return PMIX_ERROR; + } + + *priority = 30; + *module = (pmix_mca_base_module_t *)&pmix_ds21_module; + return PMIX_SUCCESS; +} + + +static int component_close(void) +{ + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_file.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_file.c new file mode 100644 index 0000000000..7a23edd206 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_file.c @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include "src/include/pmix_globals.h" +#include "src/mca/gds/base/base.h" + +#include "src/mca/common/dstore/dstore_file.h" +#include "gds_ds21_file.h" + +#if 8 > SIZEOF_SIZE_T +#define ESH_REGION_EXTENSION_FLG 0x80000000 +#define ESH_REGION_INVALIDATED_FLG 0x40000000 +#define ESH_REGION_SIZE_MASK 0x3FFFFFFF +#else +#define ESH_REGION_EXTENSION_FLG 0x8000000000000000 +#define ESH_REGION_INVALIDATED_FLG 0x4000000000000000 +#define ESH_REGION_SIZE_MASK 0x3FFFFFFFFFFFFFFF +#endif + +#define ESH_KV_SIZE_V21(addr) \ +__pmix_attribute_extension__ ({ \ + size_t sz; \ + memcpy(&sz, addr, sizeof(size_t)); \ + /* drop flags in lsb's */ \ + (sz & ESH_REGION_SIZE_MASK); \ +}) + +#define ESH_KNAME_PTR_V21(addr) \ + ((char *)addr + 2 * sizeof(size_t)) + +#define ESH_KNAME_LEN_V21(key) \ + (strlen(key) + 1) + +#define ESH_DATA_PTR_V21(addr) \ +__pmix_attribute_extension__ ({ \ + char *key_ptr = ESH_KNAME_PTR_V21(addr); \ + size_t kname_len = ESH_KNAME_LEN_V21(key_ptr); \ + uint8_t *data_ptr = \ + addr + (key_ptr - (char*)addr) + kname_len; \ + data_ptr; \ +}) + +#define ESH_DATA_SIZE_V21(addr, data_ptr) \ +__pmix_attribute_extension__ ({ \ + size_t sz = ESH_KV_SIZE_V21(addr); \ + size_t data_size = sz - (data_ptr - addr); \ + data_size; \ +}) + +#define ESH_KEY_SIZE_V21(key, size) \ + (2 * sizeof(size_t) + ESH_KNAME_LEN_V21((char*)key) + size) + +/* in ext slot new offset will be stored in case if + * new data were added for the same process during + * next commit + */ +#define EXT_SLOT_SIZE_V21() \ + (ESH_KEY_SIZE_V21("", sizeof(size_t))) + +static bool pmix_ds21_is_invalid(uint8_t *addr) +{ + size_t sz; + memcpy(&sz, addr, sizeof(size_t)); + return !!(sz & ESH_REGION_INVALIDATED_FLG); +} + +static void pmix_ds21_set_invalid(uint8_t *addr) +{ + size_t sz; + memcpy(&sz, addr, sizeof(size_t)); + sz |= ESH_REGION_INVALIDATED_FLG; + memcpy(addr, &sz, sizeof(size_t)); +} + +static bool pmix_ds21_is_ext_slot(uint8_t *addr) +{ + size_t sz; + memcpy(&sz, addr, sizeof(size_t)); + return !!(sz & ESH_REGION_EXTENSION_FLG); +} + +static size_t pmix_ds21_key_hash(const char *key) +{ + size_t hash = 0; + int i; + for(i=0; key[i]; i++) { + hash += key[i]; + } + return hash; +} + +static bool pmix_ds21_kname_match(uint8_t *addr, const char *key, size_t key_hash) +{ + bool ret = 0; + size_t hash; + memcpy(&hash, (char*)addr + sizeof(size_t), sizeof(size_t)); + if( key_hash != hash ) { + return ret; + } + return (0 == strncmp(ESH_KNAME_PTR_V21(addr), key, ESH_KNAME_LEN_V21(key))); +} + +static size_t pmix_ds21_kval_size(uint8_t *key) +{ + return ESH_KV_SIZE_V21(key); ; +} + +static char* pmix_ds21_key_name_ptr(uint8_t *addr) +{ + return ESH_KNAME_PTR_V21(addr); +} + +static size_t pmix_ds21_key_name_len(char *key) +{ + return ESH_KNAME_LEN_V21(key); +} + +static uint8_t* pmix_ds21_data_ptr(uint8_t *addr) +{ + return ESH_DATA_PTR_V21(addr); +} + +static size_t pmix_ds21_data_size(uint8_t *addr, uint8_t* data_ptr) +{ + return ESH_DATA_SIZE_V21(addr, data_ptr); +} + +static size_t pmix_ds21_key_size(char *addr, size_t data_size) +{ + return ESH_KEY_SIZE_V21(addr, data_size); +} + +static size_t pmix_ds21_ext_slot_size(void) +{ + return EXT_SLOT_SIZE_V21(); +} + +static int pmix_ds21_put_key(uint8_t *addr, char *key, + void* buffer, size_t size) +{ + size_t flag = 0; + size_t hash = 0; + char *addr_ch = (char*)addr; + if( !strcmp(key, ESH_REGION_EXTENSION) ) { + /* we have a flag for this special key */ + key = ""; + flag |= ESH_REGION_EXTENSION_FLG; + } + size_t sz = ESH_KEY_SIZE_V21(key, size); + if( ESH_REGION_SIZE_MASK < sz ) { + return PMIX_ERROR; + } + sz |= flag; + memcpy(addr_ch, &sz, sizeof(size_t)); + hash = pmix_ds21_key_hash(key); + memcpy(addr_ch + sizeof(size_t), &hash, sizeof(size_t)); + strncpy(addr_ch + 2 * sizeof(size_t), key, ESH_KNAME_LEN_V21(key)); + memcpy(ESH_DATA_PTR_V21(addr), buffer, size); + return PMIX_SUCCESS; +} + +pmix_common_dstore_file_cbs_t pmix_ds21_file_module = { + .name = "ds21", + .kval_size = pmix_ds21_kval_size, + .kname_ptr = pmix_ds21_key_name_ptr, + .kname_len = pmix_ds21_key_name_len, + .data_ptr = pmix_ds21_data_ptr, + .data_size = pmix_ds21_data_size, + .key_size = pmix_ds21_key_size, + .ext_slot_size = pmix_ds21_ext_slot_size, + .put_key = pmix_ds21_put_key, + .is_invalid = pmix_ds21_is_invalid, + .is_extslot = pmix_ds21_is_ext_slot, + .set_invalid = pmix_ds21_set_invalid, + .key_hash = pmix_ds21_key_hash, + .key_match = pmix_ds21_kname_match +}; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_file.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_file.h new file mode 100644 index 0000000000..ea75788ffe --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_file.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef GDS_DS21_FILE_H +#define GDS_DS21_FILE_H + +#include +#include + +extern pmix_common_dstore_file_cbs_t pmix_ds21_file_module; + +#endif // GDS_DS21_FILE_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock.c new file mode 100644 index 0000000000..340343d8c8 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock.c @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include "src/mca/common/dstore/dstore_common.h" + +#include "gds_ds21_lock.h" + +pmix_common_lock_callbacks_t pmix_ds21_lock_module = { + .init = pmix_gds_ds21_lock_init, + .finalize = pmix_ds21_lock_finalize, + .rd_lock = pmix_ds21_lock_rd_get, + .rd_unlock = pmix_ds21_lock_rd_rel, + .wr_lock = pmix_ds21_lock_wr_get, + .wr_unlock = pmix_ds21_lock_wr_rel +}; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock.h new file mode 100644 index 0000000000..158e7cbf5a --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef DS21_LOCK_H +#define DS21_LOCK_H + +#include +#include + +#include "src/mca/common/dstore/dstore_common.h" + +pmix_status_t pmix_gds_ds21_lock_init(pmix_common_dstor_lock_ctx_t *lock_ctx, + const char *base_path, const char *name, + uint32_t local_size, uid_t uid, bool setuid); +void pmix_ds21_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx); +pmix_status_t pmix_ds21_lock_rd_get(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds21_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds21_lock_rd_rel(pmix_common_dstor_lock_ctx_t lock_ctx); +pmix_status_t pmix_ds21_lock_wr_rel(pmix_common_dstor_lock_ctx_t lock_ctx); + +extern pmix_common_lock_callbacks_t pmix_ds21_lock_module; + +#endif // DS21_LOCK_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c new file mode 100644 index 0000000000..5e8b7be92b --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/ds21/gds_ds21_lock_pthread.c @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#include "src/mca/common/dstore/dstore_common.h" +#include "src/mca/gds/base/base.h" +#include "src/mca/pshmem/pshmem.h" +#include "src/class/pmix_list.h" + +#include "src/util/error.h" +#include "src/util/output.h" + +#include "gds_ds21_lock.h" +#include "src/mca/common/dstore/dstore_segment.h" + +typedef struct { + pmix_list_item_t super; + + char *lockfile; + pmix_dstore_seg_desc_t *seg_desc; + pthread_mutex_t *mutex; + uint32_t num_locks; + uint32_t lock_idx; +} lock_item_t; + +typedef struct { + pmix_list_t lock_traker; +} lock_ctx_t; + +typedef pmix_list_t ds21_lock_pthread_ctx_t; + +/* + * Lock segment format: + * 1. Segment size sizeof(size_t) + * 2. local_size: sizeof(uint32_t) + * 3. Align size sizeof(size_t) + * 4. Offset of mutexes sizeof(size_t) + * 5. Array of in use indexes: sizeof(int32_t)*local_size + * 6. Double array of locks: sizeof(pthread_mutex_t)*local_size*2 + */ +typedef struct { + size_t seg_size; + uint32_t num_locks; + size_t align_size; + size_t mutex_offs; +} segment_hdr_t; + +#define _GET_IDX_ARR_PTR(seg_ptr) \ + ((pmix_atomic_int32_t*)((char*)seg_ptr + sizeof(segment_hdr_t))) + +#define _GET_MUTEX_ARR_PTR(seg_hdr) \ + ((pthread_mutex_t*)((char*)seg_hdr + seg_hdr->mutex_offs)) + +#define _GET_MUTEX_PTR(seg_hdr, idx) \ + ((pthread_mutex_t*)((char*)seg_hdr + seg_hdr->mutex_offs + seg_hdr->align_size * (idx))) + + +static void ncon(lock_item_t *p) { + p->lockfile = NULL; + p->lock_idx = 0; + p->mutex = NULL; + p->num_locks = 0; + p->seg_desc = NULL; +} + +static void ldes(lock_item_t *p) { + uint32_t i; + + if(PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + segment_hdr_t *seg_hdr = (segment_hdr_t *)p->seg_desc->seg_info.seg_base_addr; + if (p->lockfile) { + unlink(p->lockfile); + } + for(i = 0; i < p->num_locks * 2; i++) { + pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, i); + if (0 != pthread_mutex_destroy(mutex)) { + PMIX_ERROR_LOG(PMIX_ERROR); + } + } + } + if (p->lockfile) { + free(p->lockfile); + } + if (p->seg_desc) { + pmix_common_dstor_delete_sm_desc(p->seg_desc); + } +} + +PMIX_CLASS_INSTANCE(lock_item_t, + pmix_list_item_t, + ncon, ldes); + +pmix_status_t pmix_gds_ds21_lock_init(pmix_common_dstor_lock_ctx_t *ctx, const char *base_path, const char * name, + uint32_t local_size, uid_t uid, bool setuid) +{ + pthread_mutexattr_t attr; + size_t size; + uint32_t i; + int page_size = pmix_common_dstor_getpagesize(); + segment_hdr_t *seg_hdr; + lock_item_t *lock_item = NULL; + lock_ctx_t *lock_ctx = (lock_ctx_t*)*ctx; + pmix_list_t *lock_tracker; + pmix_status_t rc = PMIX_SUCCESS; + + if (NULL == *ctx) { + lock_ctx = (lock_ctx_t*)malloc(sizeof(lock_ctx_t)); + if (NULL == lock_ctx) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + memset(lock_ctx, 0, sizeof(lock_ctx_t)); + PMIX_CONSTRUCT(&lock_ctx->lock_traker, pmix_list_t); + *ctx = lock_ctx; + } + + lock_tracker = &lock_ctx->lock_traker; + lock_item = PMIX_NEW(lock_item_t); + + if (NULL == lock_item) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + pmix_list_append(lock_tracker, &lock_item->super); + + PMIX_OUTPUT_VERBOSE((10, pmix_gds_base_framework.framework_output, + "%s:%d:%s local_size %d", __FILE__, __LINE__, __func__, local_size)); + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + size_t seg_align_size; + size_t seg_hdr_size; + + if (0 != (seg_align_size = pmix_common_dstor_getcacheblocksize())) { + seg_align_size = (sizeof(pthread_mutex_t) / seg_align_size + 1) + * seg_align_size; + } else { + seg_align_size = sizeof(pthread_mutex_t); + } + + seg_hdr_size = ((sizeof(segment_hdr_t) + + sizeof(int32_t) * local_size) + / seg_align_size + 1) * seg_align_size; + + size = ((seg_hdr_size + + 2 * local_size * seg_align_size) /* array of mutexes */ + / page_size + 1) * page_size; + + lock_item->seg_desc = pmix_common_dstor_create_new_lock_seg(base_path, + size, name, 0, uid, setuid); + if (NULL == lock_item->seg_desc) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + PMIX_ERROR_LOG(rc); + goto error; + } + + if (0 != pthread_mutexattr_init(&attr)) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + if (0 != pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { + pthread_mutexattr_destroy(&attr); + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + + segment_hdr_t *seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr; + seg_hdr->num_locks = local_size; + seg_hdr->seg_size = size; + seg_hdr->align_size = seg_align_size; + seg_hdr->mutex_offs = seg_hdr_size; + + lock_item->lockfile = strdup(lock_item->seg_desc->seg_info.seg_name); + lock_item->num_locks = local_size; + lock_item->mutex = _GET_MUTEX_ARR_PTR(seg_hdr); + + for(i = 0; i < local_size * 2; i++) { + pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, i); + if (0 != pthread_mutex_init(mutex, &attr)) { + pthread_mutexattr_destroy(&attr); + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(rc); + goto error; + } + } + if (0 != pthread_mutexattr_destroy(&attr)) { + rc = PMIX_ERR_INIT; + PMIX_ERROR_LOG(PMIX_ERR_INIT); + goto error; + } + } + else { + pmix_atomic_int32_t *lock_idx_ptr; + bool idx_found = false; + + size = pmix_common_dstor_getpagesize(); + lock_item->seg_desc = pmix_common_dstor_attach_new_lock_seg(base_path, size, name, 0); + if (NULL == lock_item->seg_desc) { + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + seg_hdr = (segment_hdr_t*)lock_item->seg_desc->seg_info.seg_base_addr; + + if (seg_hdr->seg_size > size) { + size = seg_hdr->seg_size; + pmix_common_dstor_delete_sm_desc(lock_item->seg_desc); + lock_item->seg_desc = pmix_common_dstor_attach_new_lock_seg(base_path, size, name, 0); + if (NULL == lock_item->seg_desc) { + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + } + + lock_item->num_locks = seg_hdr->num_locks; + lock_idx_ptr = _GET_IDX_ARR_PTR(seg_hdr); + lock_item->mutex = _GET_MUTEX_ARR_PTR(seg_hdr); + + for (i = 0; i < lock_item->num_locks; i++) { + int32_t expected = 0; + if (pmix_atomic_compare_exchange_strong_32(&lock_idx_ptr[i], &expected, 1)) { + lock_item->lock_idx = i; + lock_item->lockfile = strdup(lock_item->seg_desc->seg_info.seg_name); + idx_found = true; + break; + } + } + + if (false == idx_found) { + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + } + + return rc; + +error: + if (NULL != lock_item) { + pmix_list_remove_item(lock_tracker, &lock_item->super); + PMIX_RELEASE(lock_item); + lock_item = NULL; + } + *ctx = NULL; + + return rc; +} + +void pmix_ds21_lock_finalize(pmix_common_dstor_lock_ctx_t *lock_ctx) +{ + lock_item_t *lock_item, *item_next; + pmix_list_t *lock_tracker = &((lock_ctx_t*)*lock_ctx)->lock_traker; + + if (NULL == lock_tracker) { + return; + } + + PMIX_LIST_FOREACH_SAFE(lock_item, item_next, lock_tracker, lock_item_t) { + pmix_list_remove_item(lock_tracker, &lock_item->super); + PMIX_RELEASE(lock_item); + } + if (pmix_list_is_empty(lock_tracker)) { + PMIX_LIST_DESTRUCT(lock_tracker); + free(lock_tracker); + lock_tracker = NULL; + } + *lock_ctx = NULL; +} + +pmix_status_t pmix_ds21_lock_wr_get(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + lock_item_t *lock_item; + pmix_list_t *lock_tracker = &((lock_ctx_t*)lock_ctx)->lock_traker; + uint32_t num_locks; + uint32_t i; + pmix_status_t rc; + segment_hdr_t *seg_hdr; + + if (NULL == lock_tracker) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); + return rc; + } + + PMIX_LIST_FOREACH(lock_item, lock_tracker, lock_item_t) { + num_locks = lock_item->num_locks; + seg_hdr = (segment_hdr_t *)lock_item->seg_desc->seg_info.seg_base_addr; + + /* Lock the "signalling" lock first to let clients know that + * server is going to get a write lock. + * Clients do not hold this lock for a long time, + * so this loop should be relatively dast. + */ + for (i = 0; i < num_locks; i++) { + pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, 2*i); + if (0 != pthread_mutex_lock(mutex)) { + return PMIX_ERROR; + } + } + + /* Now we can go and grab the main locks + * New clients will be stopped at the previous + * "barrier" locks. + * We will wait here while all clients currently holding + * locks will be done + */ + for(i = 0; i < num_locks; i++) { + pthread_mutex_t *mutex = _GET_MUTEX_PTR(seg_hdr, 2*i + 1); + if (0 != pthread_mutex_lock(mutex)) { + return PMIX_ERROR; + } + } + } + return PMIX_SUCCESS; +} + +pmix_status_t pmix_ds21_lock_wr_rel(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + lock_item_t *lock_item; + pmix_list_t *lock_tracker = &((lock_ctx_t*)lock_ctx)->lock_traker; + uint32_t num_locks; + uint32_t i; + pmix_status_t rc; + segment_hdr_t *seg_hdr; + + if (NULL == lock_tracker) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + + PMIX_LIST_FOREACH(lock_item, lock_tracker, lock_item_t) { + seg_hdr = (segment_hdr_t *)lock_item->seg_desc->seg_info.seg_base_addr; + num_locks = lock_item->num_locks; + + /* Lock the second lock first to ensure that all procs will see + * that we are trying to grab the main one */ + for(i=0; ilock_traker; + uint32_t idx; + pmix_status_t rc; + segment_hdr_t *seg_hdr; + + if (NULL == lock_tracker) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + + lock_item = (lock_item_t*)pmix_list_get_first(lock_tracker); + idx = lock_item->lock_idx; + seg_hdr = (segment_hdr_t *)lock_item->seg_desc->seg_info.seg_base_addr; + + /* This mutex is only used to acquire the next one, + * this is a barrier that server is using to let clients + * know that it is going to grab the write lock + */ + + if (0 != pthread_mutex_lock(_GET_MUTEX_PTR(seg_hdr, 2*idx))) { + return PMIX_ERROR; + } + + /* Now grab the main lock */ + if (0 != pthread_mutex_lock(_GET_MUTEX_PTR(seg_hdr, 2*idx + 1))) { + return PMIX_ERROR; + } + + /* Once done - release signalling lock */ + if (0 != pthread_mutex_unlock(_GET_MUTEX_PTR(seg_hdr, 2*idx))) { + return PMIX_ERROR; + } + + return PMIX_SUCCESS; +} + +pmix_status_t pmix_ds21_lock_rd_rel(pmix_common_dstor_lock_ctx_t lock_ctx) +{ + lock_item_t *lock_item; + pmix_list_t *lock_tracker = &((lock_ctx_t*)lock_ctx)->lock_traker; + pmix_status_t rc; + uint32_t idx; + segment_hdr_t *seg_hdr; + + if (NULL == lock_tracker) { + rc = PMIX_ERR_NOT_FOUND; + PMIX_ERROR_LOG(rc); + return rc; + } + + lock_item = (lock_item_t*)pmix_list_get_first(lock_tracker); + seg_hdr = (segment_hdr_t *)lock_item->seg_desc->seg_info.seg_base_addr; + idx = lock_item->lock_idx; + + /* Release the main lock */ + if (0 != pthread_mutex_unlock(_GET_MUTEX_PTR(seg_hdr, 2*idx + 1))) { + return PMIX_SUCCESS; + } + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/gds.h b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/gds.h index 9d90dbc565..9ced4788aa 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/gds.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/gds.h @@ -1,8 +1,9 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2016-2017 Mellanox Technologies, Inc. + * Copyright (c) 2016-2018 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -44,7 +45,7 @@ BEGIN_C_DECLS /* forward declaration */ struct pmix_peer_t; -struct pmix_nspace_t; +struct pmix_namespace_t; /* backdoor to base verbosity */ PMIX_EXPORT extern int pmix_gds_base_output; @@ -117,7 +118,7 @@ typedef pmix_status_t (*pmix_gds_base_module_accept_kvs_resp_fn_t)(pmix_buffer_t * only we don't have packed data on the server side, and don't want * to incur the overhead of packing it just to unpack it in the function. */ -typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_nspace_t *ns, +typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_namespace_t *ns, pmix_info_t info[], size_t ninfo); /* define a convenience macro for caching job info */ @@ -127,7 +128,7 @@ typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_ns pmix_output_verbose(1, pmix_gds_base_output, \ "[%s:%d] GDS CACHE JOB INFO WITH %s", \ __FILE__, __LINE__, _g->name); \ - (s) = _g->cache_job_info((struct pmix_nspace_t*)(n), (i), (ni)); \ + (s) = _g->cache_job_info((struct pmix_namespace_t*)(n), (i), (ni)); \ } while(0) /* register job-level info - this is provided as a special function @@ -135,7 +136,7 @@ typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_ns * prepare the job-level info provided at PMIx_Register_nspace, because * we don't know the GDS component to use for that application until * a local client contacts us. Thus, the module is required to process - * the job-level info cached in the pmix_nspace_t for this job and + * the job-level info cached in the pmix_namespace_t for this job and * do whatever is necessary to support the client, packing any required * return message into the provided buffer. * @@ -155,7 +156,7 @@ typedef pmix_status_t (*pmix_gds_base_module_cache_job_info_fn_t)(struct pmix_ns * * The pmix_peer_t of the requesting client is provided here so that * the module can access the job-level info cached on the corresponding - * pmix_nspace_t pointed to by the pmix_peer_t + * pmix_namespace_t pointed to by the pmix_peer_t */ typedef pmix_status_t (*pmix_gds_base_module_register_job_info_fn_t)(struct pmix_peer_t *pr, pmix_buffer_t *reply); @@ -241,16 +242,16 @@ typedef pmix_status_t (*pmix_gds_base_module_store_fn_t)(const pmix_proc_t *proc * bo - pointer to the byte object containing the data * */ -typedef pmix_status_t (*pmix_gds_base_module_store_modex_fn_t)(struct pmix_nspace_t *ns, +typedef pmix_status_t (*pmix_gds_base_module_store_modex_fn_t)(struct pmix_namespace_t *ns, pmix_list_t *cbs, - pmix_byte_object_t *bo); + pmix_buffer_t *buff); /** * define a convenience macro for storing modex byte objects * * r - return status code * - * n - pointer to the pmix_nspace_t this blob is to be stored for + * n - pointer to the pmix_namespace_t this blob is to be stored for * * l - pointer to pmix_list_t containing pmix_server_caddy_t objects * of the local_cbs of the collective tracker @@ -262,7 +263,7 @@ typedef pmix_status_t (*pmix_gds_base_module_store_modex_fn_t)(struct pmix_nspac pmix_output_verbose(1, pmix_gds_base_output, \ "[%s:%d] GDS STORE MODEX WITH %s", \ __FILE__, __LINE__, (n)->compat.gds->name); \ - (r) = (n)->compat.gds->store_modex((struct pmix_nspace_t*)n, l, b); \ + (r) = (n)->compat.gds->store_modex((struct pmix_namespace_t*)n, l, b); \ } while (0) /** @@ -398,12 +399,26 @@ typedef pmix_status_t (*pmix_gds_base_module_del_nspace_fn_t)(const char* nspace } \ } while(0) +/* define a convenience macro for is_tsafe for fetch operation */ +#define PMIX_GDS_FETCH_IS_TSAFE(s, p) \ + do { \ + pmix_gds_base_module_t *_g = (p)->nptr->compat.gds; \ + pmix_output_verbose(1, pmix_gds_base_output, \ + "[%s:%d] GDS FETCH IS THREAD SAFE WITH %s", \ + __FILE__, __LINE__, _g->name); \ + if (true == _g->is_tsafe) { \ + (s) = PMIX_SUCCESS; \ + } else { \ + (s) = PMIX_ERR_NOT_SUPPORTED; \ + } \ +} while(0) /** * structure for gds modules */ typedef struct { const char *name; + const bool is_tsafe; pmix_gds_base_module_init_fn_t init; pmix_gds_base_module_fini_fn_t finalize; pmix_gds_base_assign_module_fn_t assign_module; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c index da9608fb3e..4b02d8faf2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/gds/hash/gds_hash.c @@ -1,8 +1,10 @@ /* * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. * * $COPYRIGHT$ * @@ -51,7 +53,7 @@ static void hash_finalize(void); static pmix_status_t hash_assign_module(pmix_info_t *info, size_t ninfo, int *priority); -static pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, +static pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, pmix_info_t info[], size_t ninfo); static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, @@ -64,9 +66,14 @@ static pmix_status_t hash_store(const pmix_proc_t *proc, pmix_scope_t scope, pmix_kval_t *kv); -static pmix_status_t hash_store_modex(struct pmix_nspace_t *ns, +static pmix_status_t hash_store_modex(struct pmix_namespace_t *ns, pmix_list_t *cbs, - pmix_byte_object_t *bo); + pmix_buffer_t *buff); + +static pmix_status_t _hash_store_modex(void * cbdata, + struct pmix_namespace_t *ns, + pmix_list_t *cbs, + pmix_byte_object_t *bo); static pmix_status_t hash_fetch(const pmix_proc_t *proc, pmix_scope_t scope, bool copy, @@ -91,6 +98,7 @@ static pmix_status_t accept_kvs_resp(pmix_buffer_t *buf); pmix_gds_base_module_t pmix_hash_module = { .name = "hash", + .is_tsafe = false, .init = hash_init, .finalize = hash_finalize, .assign_module = hash_assign_module, @@ -110,7 +118,7 @@ pmix_gds_base_module_t pmix_hash_module = { typedef struct { pmix_list_item_t super; char *ns; - pmix_nspace_t *nptr; + pmix_namespace_t *nptr; pmix_hash_table_t internal; pmix_hash_table_t remote; pmix_hash_table_t local; @@ -355,10 +363,10 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, return PMIX_SUCCESS; } -pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, +pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, pmix_info_t info[], size_t ninfo) { - pmix_nspace_t *nptr = (pmix_nspace_t*)ns; + pmix_namespace_t *nptr = (pmix_namespace_t*)ns; pmix_hash_trkr_t *trk, *t; pmix_hash_table_t *ht; pmix_kval_t *kp2, *kvptr; @@ -580,7 +588,7 @@ pmix_status_t hash_cache_job_info(struct pmix_nspace_t *ns, } static pmix_status_t register_info(pmix_peer_t *peer, - pmix_nspace_t *ns, + pmix_namespace_t *ns, pmix_buffer_t *reply) { pmix_hash_trkr_t *trk, *t; @@ -672,13 +680,13 @@ static pmix_status_t register_info(pmix_peer_t *peer, } /* the purpose of this function is to pack the job-level - * info stored in the pmix_nspace_t into a buffer and send + * info stored in the pmix_namespace_t into a buffer and send * it to the given client */ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, pmix_buffer_t *reply) { pmix_peer_t *peer = (pmix_peer_t*)pr; - pmix_nspace_t *ns = peer->nptr; + pmix_namespace_t *ns = peer->nptr; char *msg; pmix_status_t rc; pmix_hash_trkr_t *trk, *t2; @@ -1181,11 +1189,18 @@ static pmix_status_t hash_store(const pmix_proc_t *proc, * host has received data from some other peer. It therefore * always contains data solely from remote procs, and we * shall store it accordingly */ -static pmix_status_t hash_store_modex(struct pmix_nspace_t *nspace, +static pmix_status_t hash_store_modex(struct pmix_namespace_t *nspace, pmix_list_t *cbs, - pmix_byte_object_t *bo) + pmix_buffer_t *buf) { + return pmix_gds_base_store_modex(nspace, cbs, buf, _hash_store_modex, NULL); +} + +static pmix_status_t _hash_store_modex(void * cbdata, + struct pmix_namespace_t *nspace, + pmix_list_t *cbs, + pmix_byte_object_t *bo) { - pmix_nspace_t *ns = (pmix_nspace_t*)nspace; + pmix_namespace_t *ns = (pmix_namespace_t*)nspace; pmix_hash_trkr_t *trk, *t; pmix_status_t rc = PMIX_SUCCESS; int32_t cnt; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c index 9157d54661..800923c34f 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv4/pif_bsdx.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,23 +36,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -173,7 +158,7 @@ static int if_bsdx_open(void) /* fill values into the pmix_pif_t */ memcpy(&a4, &(sin_addr->sin_addr), sizeof(struct in_addr)); - strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE); + pmix_strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE-1); intf->if_index = pmix_list_get_size(&pmix_if_list) + 1; ((struct sockaddr_in*) &intf->if_addr)->sin_addr = a4; ((struct sockaddr_in*) &intf->if_addr)->sin_family = AF_INET; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c index 2dac2550d3..5954f1580c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/bsdx_ipv6/pif_bsdx_ipv6.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,23 +36,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -198,7 +183,7 @@ static int if_bsdx_ipv6_open(void) return PMIX_ERR_OUT_OF_RESOURCE; } intf->af_family = AF_INET6; - strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE); + pmix_strncpy(intf->if_name, cur_ifaddrs->ifa_name, IF_NAMESIZE-1); intf->if_index = pmix_list_get_size(&pmix_if_list) + 1; ((struct sockaddr_in6*) &intf->if_addr)->sin6_addr = a6; ((struct sockaddr_in6*) &intf->if_addr)->sin6_family = AF_INET6; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c index 2f240f9d8a..f0bb2db9f5 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c @@ -1,6 +1,7 @@ /* * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,23 +36,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -142,7 +127,7 @@ static int if_linux_ipv6_open(void) } /* now construct the pmix_pif_t */ - strncpy(intf->if_name, ifname, IF_NAMESIZE); + pmix_strncpy(intf->if_name, ifname, IF_NAMESIZE-1); intf->if_index = pmix_list_get_size(&pmix_if_list)+1; intf->if_kernel_index = (uint16_t) idx; ((struct sockaddr_in6*) &intf->if_addr)->sin6_addr = a6; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/pif.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/pif.h index 29c75b869c..e43de4707d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/pif.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/pif.h @@ -3,7 +3,7 @@ * Copyright (c) 2010-2013 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -40,23 +40,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c index c338b4f86a..095a3027e7 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/posix_ipv4/pif_posix.c @@ -39,23 +39,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -265,7 +249,7 @@ static int if_posix_open(void) /* copy entry over into our data structure */ memset(intf->if_name, 0, sizeof(intf->if_name)); - strncpy(intf->if_name, ifr->ifr_name, sizeof(intf->if_name) - 1); + pmix_strncpy(intf->if_name, ifr->ifr_name, sizeof(intf->if_name) - 1); intf->if_flags = ifr->ifr_flags; /* every new address gets its own internal if_index */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c index 7403cebf0e..a7d94f7908 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pif/solaris_ipv6/pif_solaris_ipv6.c @@ -3,7 +3,7 @@ * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -146,8 +146,8 @@ static int if_solaris_ipv6_open(void) i += sizeof (*lifreq)) { lifreq = (struct lifreq *)((caddr_t)lifconf.lifc_buf + i); - strncpy (lifquery.lifr_name, lifreq->lifr_name, - sizeof (lifquery.lifr_name)); + pmix_strncpy (lifquery.lifr_name, lifreq->lifr_name, + sizeof (lifquery.lifr_name)-1); /* lookup kernel index */ error = ioctl (sd, SIOCGLIFINDEX, &lifquery); @@ -190,7 +190,7 @@ static int if_solaris_ipv6_open(void) } intf->af_family = AF_INET6; - strncpy (intf->if_name, lifreq->lifr_name, IF_NAMESIZE); + pmix_strncpy (intf->if_name, lifreq->lifr_name, IF_NAMESIZE-1); intf->if_index = pmix_list_get_size(&pmix_if_list)+1; memcpy(&intf->if_addr, my_addr, sizeof (*my_addr)); intf->if_mask = 64; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pinstalldirs/config/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/mca/pinstalldirs/config/Makefile.am index d05743fb5f..7a1f9c9c3d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pinstalldirs/config/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pinstalldirs/config/Makefile.am @@ -4,7 +4,7 @@ # Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2009 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. -# Copyright (c) 2016 Intel, Inc. All rights reserved. +# Copyright (c) 2016-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -19,4 +19,4 @@ libmca_pinstalldirs_config_la_SOURCES = \ # This file is generated; we do not want to include it in the tarball nodist_libmca_pinstalldirs_config_la_SOURCES = \ - install_dirs.h + pinstall_dirs.h diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c index e6ed5a60ce..619dc38f70 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/plog/stdfd/plog_stdfd.c @@ -90,7 +90,7 @@ static pmix_status_t mylog(const pmix_proc_t *source, /* check to see if there are any relevant directives */ for (n=0; n < ndirs; n++) { if (0 == strncmp(directives[n].key, PMIX_LOG_TIMESTAMP, PMIX_MAX_KEYLEN)) { - flags.timestamp = data[n].value.data.time; + flags.timestamp = directives[n].value.data.time; } else if (0 == strncmp(directives[n].key, PMIX_LOG_XML_OUTPUT, PMIX_MAX_KEYLEN)) { flags.xml = PMIX_INFO_TRUE(&directives[n]); } else if (0 == strncmp(directives[n].key, PMIX_LOG_TAG_OUTPUT, PMIX_MAX_KEYLEN)) { diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/base.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/base.h index 9b4f58fdaf..d832bf5478 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/base.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/base.h @@ -119,7 +119,7 @@ PMIX_EXPORT pmix_status_t pmix_pnet_base_setup_local_network(char *nspace, size_t ninfo); PMIX_EXPORT pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *peer, char ***env); PMIX_EXPORT void pmix_pnet_base_child_finalized(pmix_proc_t *peer); -PMIX_EXPORT void pmix_pnet_base_local_app_finalized(pmix_nspace_t *nptr); +PMIX_EXPORT void pmix_pnet_base_local_app_finalized(pmix_namespace_t *nptr); PMIX_EXPORT void pmix_pnet_base_deregister_nspace(char *nspace); PMIX_EXPORT void pmix_pnet_base_collect_inventory(pmix_info_t directives[], size_t ndirs, pmix_inventory_cbfunc_t cbfunc, diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_fns.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_fns.c index 22ea10829e..c4869da529 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_fns.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/base/pnet_base_fns.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2018 Research Organization for Information Science @@ -39,10 +39,11 @@ pmix_status_t pmix_pnet_base_allocate(char *nspace, pmix_list_t *ilist) { pmix_pnet_base_active_module_t *active; - pmix_status_t rc; - pmix_nspace_t *nptr, *ns; + pmix_status_t rc = PMIX_SUCCESS; + pmix_namespace_t *nptr, *ns; size_t n; char *nregex, *pregex; + char *params[2] = {"PMIX_MCA_", NULL}; if (!pmix_pnet_globals.initialized) { return PMIX_ERR_INIT; @@ -59,7 +60,7 @@ pmix_status_t pmix_pnet_base_allocate(char *nspace, nptr = NULL; /* find this nspace - note that it may not have * been registered yet */ - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(ns->nspace, nspace)) { nptr = ns; break; @@ -67,7 +68,7 @@ pmix_status_t pmix_pnet_base_allocate(char *nspace, } if (NULL == nptr) { /* add it */ - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); if (NULL == nptr) { return PMIX_ERR_NOMEM; } @@ -75,22 +76,7 @@ pmix_status_t pmix_pnet_base_allocate(char *nspace, pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); } - /* if the info param is NULL, then we make one pass thru the actives - * in case someone specified an allocation or collection of envars - * via MCA param */ - if (NULL == info) { - PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { - if (NULL != active->module->allocate) { - if (PMIX_SUCCESS == (rc = active->module->allocate(nptr, NULL, ilist))) { - break; - } - if (PMIX_ERR_TAKE_NEXT_OPTION != rc) { - /* true error */ - return rc; - } - } - } - } else { + if (NULL != info) { /* check for description of the node and proc maps */ nregex = NULL; pregex = NULL; @@ -131,7 +117,10 @@ pmix_status_t pmix_pnet_base_allocate(char *nspace, } } - return PMIX_SUCCESS; + /* add any local PMIx MCA params */ + rc = pmix_pnet_base_harvest_envars(params, NULL, ilist); + + return rc; } /* can only be called by a server */ @@ -141,7 +130,7 @@ pmix_status_t pmix_pnet_base_setup_local_network(char *nspace, { pmix_pnet_base_active_module_t *active; pmix_status_t rc; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; if (!pmix_pnet_globals.initialized) { return PMIX_ERR_INIT; @@ -157,7 +146,7 @@ pmix_status_t pmix_pnet_base_setup_local_network(char *nspace, /* find this proc's nspace object */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(ns->nspace, nspace)) { nptr = ns; break; @@ -165,7 +154,7 @@ pmix_status_t pmix_pnet_base_setup_local_network(char *nspace, } if (NULL == nptr) { /* add it */ - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); if (NULL == nptr) { return PMIX_ERR_NOMEM; } @@ -189,7 +178,7 @@ pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *proc, char ***env) { pmix_pnet_base_active_module_t *active; pmix_status_t rc; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; if (!pmix_pnet_globals.initialized) { return PMIX_ERR_INIT; @@ -202,7 +191,7 @@ pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *proc, char ***env) /* find this proc's nspace object */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(ns->nspace, proc->nspace)) { nptr = ns; break; @@ -210,7 +199,7 @@ pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *proc, char ***env) } if (NULL == nptr) { /* add it */ - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); if (NULL == nptr) { return PMIX_ERR_NOMEM; } @@ -252,7 +241,7 @@ void pmix_pnet_base_child_finalized(pmix_proc_t *peer) return; } -void pmix_pnet_base_local_app_finalized(pmix_nspace_t *nptr) +void pmix_pnet_base_local_app_finalized(pmix_namespace_t *nptr) { pmix_pnet_base_active_module_t *active; @@ -277,7 +266,9 @@ void pmix_pnet_base_local_app_finalized(pmix_nspace_t *nptr) void pmix_pnet_base_deregister_nspace(char *nspace) { pmix_pnet_base_active_module_t *active; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; + pmix_pnet_job_t *job; + pmix_pnet_node_t *node; if (!pmix_pnet_globals.initialized) { return; @@ -290,7 +281,7 @@ void pmix_pnet_base_deregister_nspace(char *nspace) /* find this nspace object */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(ns->nspace, nspace)) { nptr = ns; break; @@ -307,7 +298,24 @@ void pmix_pnet_base_deregister_nspace(char *nspace) } } - return; + PMIX_LIST_FOREACH(job, &pmix_pnet_globals.jobs, pmix_pnet_job_t) { + if (0 == strcmp(nspace, job->nspace)) { + pmix_list_remove_item(&pmix_pnet_globals.jobs, &job->super); + PMIX_RELEASE(job); + break; + } + } + + PMIX_LIST_FOREACH(node, &pmix_pnet_globals.nodes, pmix_pnet_node_t) { + pmix_pnet_local_procs_t *lp; + PMIX_LIST_FOREACH(lp, &node->local_jobs, pmix_pnet_local_procs_t) { + if (0 == strcmp(nspace, lp->nspace)) { + pmix_list_remove_item(&node->local_jobs, &lp->super); + PMIX_RELEASE(lp); + break; + } + } + } } static void cicbfunc(pmix_status_t status, @@ -560,6 +568,8 @@ pmix_status_t pmix_pnet_base_harvest_envars(char **incvars, char **excvars, } *string_key = '\0'; ++string_key; + pmix_output_verbose(5, pmix_pnet_base_framework.framework_output, + "pnet: adding envar %s", cs_env); PMIX_ENVAR_LOAD(&kv->value->data.envar, cs_env, string_key, ':'); pmix_list_append(ilist, &kv->super); free(cs_env); @@ -576,6 +586,8 @@ pmix_status_t pmix_pnet_base_harvest_envars(char **incvars, char **excvars, } PMIX_LIST_FOREACH_SAFE(kv, next, ilist, pmix_kval_t) { if (0 == strncmp(kv->value->data.envar.envar, excvars[j], len)) { + pmix_output_verbose(5, pmix_pnet_base_framework.framework_output, + "pnet: excluding envar %s", kv->value->data.envar.envar); pmix_list_remove_item(ilist, &kv->super); PMIX_RELEASE(kv); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c index 2bddd22f18..d795c8bc48 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * * $COPYRIGHT$ @@ -52,18 +52,18 @@ static pmix_status_t opa_init(void); static void opa_finalize(void); -static pmix_status_t allocate(pmix_nspace_t *nptr, +static pmix_status_t allocate(pmix_namespace_t *nptr, pmix_info_t *info, pmix_list_t *ilist); -static pmix_status_t setup_local_network(pmix_nspace_t *nptr, +static pmix_status_t setup_local_network(pmix_namespace_t *nptr, pmix_info_t info[], size_t ninfo); -static pmix_status_t setup_fork(pmix_nspace_t *nptr, +static pmix_status_t setup_fork(pmix_namespace_t *nptr, const pmix_proc_t *proc, char ***env); static void child_finalized(pmix_proc_t *peer); -static void local_app_finalized(pmix_nspace_t *nptr); -static void deregister_nspace(pmix_nspace_t *nptr); +static void local_app_finalized(pmix_namespace_t *nptr); +static void deregister_nspace(pmix_namespace_t *nptr); static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, pmix_inventory_cbfunc_t cbfunc, void *cbdata); static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, @@ -229,7 +229,7 @@ static char* transports_print(uint64_t *unique_key) /* NOTE: if there is any binary data to be transferred, then * this function MUST pack it for transport as the host will * not know how to do so */ -static pmix_status_t allocate(pmix_nspace_t *nptr, +static pmix_status_t allocate(pmix_namespace_t *nptr, pmix_info_t *info, pmix_list_t *ilist) { @@ -251,16 +251,19 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, return PMIX_ERR_TAKE_NEXT_OPTION; } - if (0 == strncmp(info->key, PMIX_SETUP_APP_ENVARS, PMIX_MAX_KEYLEN)) { + if (PMIX_CHECK_KEY(info, PMIX_SETUP_APP_ENVARS)) { envars = PMIX_INFO_TRUE(info); - } else if (0 == strncmp(info->key, PMIX_SETUP_APP_ALL, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(info, PMIX_SETUP_APP_ALL)) { envars = PMIX_INFO_TRUE(info); seckeys = PMIX_INFO_TRUE(info); - } else if (0 == strncmp(info->key, PMIX_SETUP_APP_NONENVARS, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(info, PMIX_SETUP_APP_NONENVARS) || + PMIX_CHECK_KEY(info, PMIX_ALLOC_NETWORK_SEC_KEY)) { seckeys = PMIX_INFO_TRUE(info); } if (seckeys) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: opa providing seckeys"); /* put the number here - or else create an appropriate string. this just needs to * eventually be a string variable */ @@ -311,6 +314,10 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, } if (envars) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: opa harvesting envars %s excluding %s", + (NULL == mca_pnet_opa_component.incparms) ? "NONE" : mca_pnet_opa_component.incparms, + (NULL == mca_pnet_opa_component.excparms) ? "NONE" : mca_pnet_opa_component.excparms); /* harvest envars to pass along */ if (NULL != mca_pnet_opa_component.include) { rc = pmix_pnet_base_harvest_envars(mca_pnet_opa_component.include, @@ -327,7 +334,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, return PMIX_ERR_TAKE_NEXT_OPTION; } -static pmix_status_t setup_local_network(pmix_nspace_t *nptr, +static pmix_status_t setup_local_network(pmix_namespace_t *nptr, pmix_info_t info[], size_t ninfo) { @@ -335,6 +342,9 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, pmix_kval_t *kv; + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: opa setup_local_network"); + if (NULL != info) { for (n=0; n < ninfo; n++) { if (0 == strncmp(info[n].key, PMIX_PNET_OPA_BLOB, PMIX_MAX_KEYLEN)) { @@ -353,6 +363,14 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, return PMIX_ERR_NOMEM; } pmix_value_xfer(kv->value, &info[n].value); + if (PMIX_ENVAR == kv->value->type) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:opa:setup_local_network adding %s=%s to environment", + kv->value->data.envar.envar, kv->value->data.envar.value); + } else { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:opa:setup_local_network loading blob"); + } pmix_list_append(&nptr->setup_data, &kv->super); } } @@ -361,12 +379,15 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, return PMIX_SUCCESS; } -static pmix_status_t setup_fork(pmix_nspace_t *nptr, +static pmix_status_t setup_fork(pmix_namespace_t *nptr, const pmix_proc_t *proc, char ***env) { pmix_kval_t *kv, *next; + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: opa setup fork"); + /* if there are any cached nspace prep blobs, execute them, * ensuring that we only do so once per nspace - note that * we don't expect to find any envars here, though we could @@ -387,14 +408,14 @@ static void child_finalized(pmix_proc_t *peer) "pnet:opa child finalized"); } -static void local_app_finalized(pmix_nspace_t *nptr) +static void local_app_finalized(pmix_namespace_t *nptr) { pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet:opa app finalized"); } -static void deregister_nspace(pmix_nspace_t *nptr) +static void deregister_nspace(pmix_namespace_t *nptr) { pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet:opa deregister nspace"); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h index 75d50d4888..6340d9f225 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -20,6 +20,8 @@ BEGIN_C_DECLS typedef struct { pmix_pnet_base_component_t super; + char *incparms; + char *excparms; char **include; char **exclude; } pmix_pnet_opa_component_t; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa_component.c index 9a726c3f4e..5ef1572239 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa_component.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/opa/pnet_opa_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -69,33 +69,30 @@ pmix_pnet_opa_component_t mca_pnet_opa_component = { .exclude = NULL }; -static char *includeparam; -static char *excludeparam; - static pmix_status_t component_register(void) { pmix_mca_base_component_t *component = &mca_pnet_opa_component.super.base; - includeparam = "HFI_*,PSM2_*"; + mca_pnet_opa_component.incparms = "HFI_*,PSM2_*"; (void)pmix_mca_base_component_var_register(component, "include_envars", "Comma-delimited list of envars to harvest (\'*\' and \'?\' supported)", PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, PMIX_INFO_LVL_2, PMIX_MCA_BASE_VAR_SCOPE_LOCAL, - &includeparam); - if (NULL != includeparam) { - mca_pnet_opa_component.include = pmix_argv_split(includeparam, ','); + &mca_pnet_opa_component.incparms); + if (NULL != mca_pnet_opa_component.incparms) { + mca_pnet_opa_component.include = pmix_argv_split(mca_pnet_opa_component.incparms, ','); } - excludeparam = NULL; + mca_pnet_opa_component.excparms = NULL; (void)pmix_mca_base_component_var_register(component, "exclude_envars", "Comma-delimited list of envars to exclude (\'*\' and \'?\' supported)", PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, PMIX_INFO_LVL_2, PMIX_MCA_BASE_VAR_SCOPE_LOCAL, - &excludeparam); - if (NULL != excludeparam) { - mca_pnet_opa_component.exclude = pmix_argv_split(excludeparam, ','); + &mca_pnet_opa_component.excparms); + if (NULL != mca_pnet_opa_component.excparms) { + mca_pnet_opa_component.exclude = pmix_argv_split(mca_pnet_opa_component.excparms, ','); } return PMIX_SUCCESS; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h index 3313ca6782..fb5cc7d363 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/pnet.h @@ -60,7 +60,7 @@ typedef void (*pmix_pnet_base_module_fini_fn_t)(void); * each other, environmental variables picked up at the login node * for forwarding to compute nodes, or allocation of static endpts */ -typedef pmix_status_t (*pmix_pnet_base_module_allocate_fn_t)(pmix_nspace_t *nptr, +typedef pmix_status_t (*pmix_pnet_base_module_allocate_fn_t)(pmix_namespace_t *nptr, pmix_info_t *info, pmix_list_t *ilist); @@ -68,7 +68,7 @@ typedef pmix_status_t (*pmix_pnet_base_module_allocate_fn_t)(pmix_nspace_t *nptr * Give the local network library an opportunity to setup address information * for the application by passing in the layout type and a regex describing * the layout */ -typedef pmix_status_t (*pmix_pnet_base_module_setup_local_net_fn_t)(pmix_nspace_t *nptr, +typedef pmix_status_t (*pmix_pnet_base_module_setup_local_net_fn_t)(pmix_namespace_t *nptr, pmix_info_t info[], size_t ninfo); @@ -76,7 +76,7 @@ typedef pmix_status_t (*pmix_pnet_base_module_setup_local_net_fn_t)(pmix_nspace_ * Give the local network library an opportunity to add any envars to the * environment of a local application process prior to fork/exec */ -typedef pmix_status_t (*pmix_pnet_base_module_setup_fork_fn_t)(pmix_nspace_t *nptr, +typedef pmix_status_t (*pmix_pnet_base_module_setup_fork_fn_t)(pmix_namespace_t *nptr, const pmix_proc_t *proc, char ***env); @@ -90,13 +90,13 @@ typedef void (*pmix_pnet_base_module_child_finalized_fn_t)(pmix_proc_t *peer); * Provide an opportunity for the local network library to cleanup after * all local clients for a given application have terminated */ -typedef void (*pmix_pnet_base_module_local_app_finalized_fn_t)(pmix_nspace_t *nptr); +typedef void (*pmix_pnet_base_module_local_app_finalized_fn_t)(pmix_namespace_t *nptr); /** * Provide an opportunity for the fabric components to cleanup any * resource allocations (e.g., static ports) they may have assigned */ -typedef void (*pmix_pnet_base_module_dregister_nspace_fn_t)(pmix_nspace_t *nptr); +typedef void (*pmix_pnet_base_module_dregister_nspace_fn_t)(pmix_namespace_t *nptr); /** @@ -166,7 +166,7 @@ typedef struct { /* define a few API versions of the functions - main difference is the - * string nspace parameter instead of a pointer to pmix_nspace_t. This + * string nspace parameter instead of a pointer to pmix_namespace_t. This * is done as an optimization to avoid having every component look for * that pointer */ typedef pmix_status_t (*pmix_pnet_base_API_allocate_fn_t)(char *nspace, diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c index 3f7a44868e..fecec01424 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.c @@ -1,5 +1,7 @@ /* - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * $COPYRIGHT$ * @@ -47,17 +49,17 @@ static pmix_status_t tcp_init(void); static void tcp_finalize(void); -static pmix_status_t allocate(pmix_nspace_t *nptr, +static pmix_status_t allocate(pmix_namespace_t *nptr, pmix_info_t *info, pmix_list_t *ilist); -static pmix_status_t setup_local_network(pmix_nspace_t *nptr, +static pmix_status_t setup_local_network(pmix_namespace_t *nptr, pmix_info_t info[], size_t ninfo); -static pmix_status_t setup_fork(pmix_nspace_t *nptr, +static pmix_status_t setup_fork(pmix_namespace_t *nptr, const pmix_proc_t *peer, char ***env); static void child_finalized(pmix_proc_t *peer); -static void local_app_finalized(pmix_nspace_t *nptr); -static void deregister_nspace(pmix_nspace_t *nptr); +static void local_app_finalized(pmix_namespace_t *nptr); +static void deregister_nspace(pmix_namespace_t *nptr); static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, pmix_inventory_cbfunc_t cbfunc, void *cbdata); static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, @@ -102,7 +104,7 @@ typedef struct { } tcp_port_tracker_t; static pmix_list_t allocations, available; -static pmix_status_t process_request(pmix_nspace_t *nptr, +static pmix_status_t process_request(pmix_namespace_t *nptr, char *idkey, int ports_per_node, tcp_port_tracker_t *trk, pmix_list_t *ilist); @@ -295,7 +297,7 @@ static inline void generate_key(uint64_t* unique_key) { * NOTE: this implementation is offered as an example that can * undoubtedly be vastly improved/optimized */ -static pmix_status_t allocate(pmix_nspace_t *nptr, +static pmix_status_t allocate(pmix_namespace_t *nptr, pmix_info_t *info, pmix_list_t *ilist) { @@ -329,16 +331,20 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, /* check directives to see if a crypto key and/or * network resource allocations requested */ PMIX_CONSTRUCT(&mylist, pmix_list_t); - if (0 == strncmp(info->key, PMIX_SETUP_APP_ENVARS, PMIX_MAX_KEYLEN) || - 0 == strncmp(info->key, PMIX_SETUP_APP_ALL, PMIX_MAX_KEYLEN)) { + if (PMIX_CHECK_KEY(info, PMIX_SETUP_APP_ENVARS) || + PMIX_CHECK_KEY(info, PMIX_SETUP_APP_ALL)) { if (NULL != mca_pnet_tcp_component.include) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: tcp harvesting envars %s excluding %s", + (NULL == mca_pnet_tcp_component.incparms) ? "NONE" : mca_pnet_tcp_component.incparms, + (NULL == mca_pnet_tcp_component.excparms) ? "NONE" : mca_pnet_tcp_component.excparms); rc = pmix_pnet_base_harvest_envars(mca_pnet_tcp_component.include, mca_pnet_tcp_component.exclude, ilist); return rc; } return PMIX_SUCCESS; - } else if (0 != strncmp(info->key, PMIX_ALLOC_NETWORK, PMIX_MAX_KEYLEN)) { + } else if (!PMIX_CHECK_KEY(info, PMIX_ALLOC_NETWORK)) { /* not a network allocation request */ return PMIX_SUCCESS; } @@ -443,11 +449,13 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, } /* nope - they asked for something that we cannot do */ if (NULL == avail) { + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOT_AVAILABLE; } /* setup to track the assignment */ trk = PMIX_NEW(tcp_port_tracker_t); if (NULL == trk) { + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOMEM; } trk->nspace = strdup(nptr->nspace); @@ -459,6 +467,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, /* return the allocated ports */ pmix_list_remove_item(&allocations, &trk->super); PMIX_RELEASE(trk); + PMIX_LIST_DESTRUCT(&mylist); return rc; } allocated = true; @@ -481,11 +490,13 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, } /* nope - they asked for something that we cannot do */ if (NULL == avail) { + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOT_AVAILABLE; } /* setup to track the assignment */ trk = PMIX_NEW(tcp_port_tracker_t); if (NULL == trk) { + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOMEM; } trk->nspace = strdup(nptr->nspace); @@ -497,6 +508,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, /* return the allocated ports */ pmix_list_remove_item(&allocations, &trk->super); PMIX_RELEASE(trk); + PMIX_LIST_DESTRUCT(&mylist); return rc; } allocated = true; @@ -505,6 +517,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet:tcp:allocate unsupported type %s for nspace %s", type, nptr->nspace); + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_TAKE_NEXT_OPTION; } @@ -519,6 +532,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, /* setup to track the assignment */ trk = PMIX_NEW(tcp_port_tracker_t); if (NULL == trk) { + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOMEM; } trk->nspace = strdup(nptr->nspace); @@ -530,6 +544,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, /* return the allocated ports */ pmix_list_remove_item(&allocations, &trk->super); PMIX_RELEASE(trk); + PMIX_LIST_DESTRUCT(&mylist); return rc; } allocated = true; @@ -583,6 +598,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, trk = PMIX_NEW(tcp_port_tracker_t); if (NULL == trk) { pmix_argv_free(reqs); + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOMEM; } trk->nspace = strdup(nptr->nspace); @@ -594,6 +610,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, /* return the allocated ports */ pmix_list_remove_item(&allocations, &trk->super); PMIX_RELEASE(trk); + PMIX_LIST_DESTRUCT(&mylist); return rc; } allocated = true; @@ -604,6 +621,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, ports_per_node, nptr->nspace); if (0 == ports_per_node) { /* nothing to allocate */ + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_TAKE_NEXT_OPTION; } avail = (tcp_available_ports_t*)pmix_list_get_first(&available); @@ -611,6 +629,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, /* setup to track the assignment */ trk = PMIX_NEW(tcp_port_tracker_t); if (NULL == trk) { + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOMEM; } trk->nspace = strdup(nptr->nspace); @@ -630,26 +649,32 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, } if (!allocated) { /* nope - we cannot help */ + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_TAKE_NEXT_OPTION; } } if (seckey) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp: generate seckey"); generate_key(unique_key); kv = PMIX_NEW(pmix_kval_t); if (NULL == kv) { + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOMEM; } kv->key = strdup(PMIX_ALLOC_NETWORK_SEC_KEY); kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); if (NULL == kv->value) { PMIX_RELEASE(kv); + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOMEM; } kv->value->type = PMIX_BYTE_OBJECT; kv->value->data.bo.bytes = (char*)malloc(2 * sizeof(uint64_t)); if (NULL == kv->value->data.bo.bytes) { PMIX_RELEASE(kv); + PMIX_LIST_DESTRUCT(&mylist); return PMIX_ERR_NOMEM; } memcpy(kv->value->data.bo.bytes, unique_key, 2 * sizeof(uint64_t)); @@ -696,7 +721,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, /* upon receipt of the launch message, each daemon adds the * static address assignments to the job-level info cache * for that job */ -static pmix_status_t setup_local_network(pmix_nspace_t *nptr, +static pmix_status_t setup_local_network(pmix_namespace_t *nptr, pmix_info_t info[], size_t ninfo) { @@ -724,7 +749,12 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &bkt, &nkvals, &cnt, PMIX_SIZE); /* setup the info array */ - PMIX_INFO_CREATE(jinfo, nkvals); + PMIX_INFO_CONSTRUCT(&stinfo); + pmix_strncpy(stinfo.key, idkey, PMIX_MAX_KEYLEN); + stinfo.value.type = PMIX_DATA_ARRAY; + PMIX_DATA_ARRAY_CREATE(stinfo.value.data.darray, nkvals, PMIX_INFO); + jinfo = (pmix_info_t*)stinfo.value.data.darray->array; + /* cycle thru the blob and extract the kvals */ kv = PMIX_NEW(pmix_kval_t); cnt = 1; @@ -736,7 +766,7 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, "recvd KEY %s %s", kv->key, (PMIX_STRING == kv->value->type) ? kv->value->data.string : "NON-STRING"); /* xfer the value to the info */ - (void)strncpy(jinfo[m].key, kv->key, PMIX_MAX_KEYLEN); + pmix_strncpy(jinfo[m].key, kv->key, PMIX_MAX_KEYLEN); PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, &jinfo[m].value, kv->value); /* if this is the ID key, save it */ @@ -762,12 +792,6 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, PMIX_INFO_FREE(jinfo, nkvals); return PMIX_ERR_BAD_PARAM; } - /* the data gets stored as a pmix_data_array_t on the provided key */ - PMIX_INFO_CONSTRUCT(&stinfo); - (void)strncpy(stinfo.key, idkey, PMIX_MAX_KEYLEN); - stinfo.value.type = PMIX_DATA_ARRAY; - PMIX_DATA_ARRAY_CREATE(stinfo.value.data.darray, nkvals, PMIX_INFO); - stinfo.value.data.darray->array = jinfo; /* cache the info on the job */ PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, @@ -782,9 +806,11 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, return PMIX_SUCCESS; } -static pmix_status_t setup_fork(pmix_nspace_t *nptr, +static pmix_status_t setup_fork(pmix_namespace_t *nptr, const pmix_proc_t *peer, char ***env) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:setup_fork"); return PMIX_SUCCESS; } @@ -801,7 +827,7 @@ static void child_finalized(pmix_proc_t *peer) * provides an opportunity for the local network to cleanup * any resources consumed locally by the clients of that job. * We don't have anything we need to do */ -static void local_app_finalized(pmix_nspace_t *nptr) +static void local_app_finalized(pmix_namespace_t *nptr) { pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet:tcp app finalized"); @@ -811,7 +837,7 @@ static void local_app_finalized(pmix_nspace_t *nptr) * PMix function, which in turn calls my TCP component to release the * assignments for that job. The addresses are marked as "available" * for reuse on the next job. */ -static void deregister_nspace(pmix_nspace_t *nptr) +static void deregister_nspace(pmix_namespace_t *nptr) { tcp_port_tracker_t *trk; @@ -852,6 +878,9 @@ static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, pmix_byte_object_t pbo; pmix_kval_t *kv; + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:tcp:collect_inventory"); + /* setup the bucket - we will pass the results as a blob */ PMIX_CONSTRUCT(&bucket, pmix_buffer_t); /* add our hostname */ @@ -949,7 +978,7 @@ static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, return PMIX_SUCCESS; } -static pmix_status_t process_request(pmix_nspace_t *nptr, +static pmix_status_t process_request(pmix_namespace_t *nptr, char *idkey, int ports_per_node, tcp_port_tracker_t *trk, pmix_list_t *ilist) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h index 63ffd87871..54e0fe0316 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -22,6 +22,8 @@ typedef struct { pmix_pnet_base_component_t super; char *static_ports; char *default_request; + char *incparms; + char *excparms; char **include; char **exclude; } pmix_pnet_tcp_component_t; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp_component.c index 64dc93c240..b313ab3607 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp_component.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/tcp/pnet_tcp_component.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -60,9 +60,6 @@ pmix_pnet_tcp_component_t mca_pnet_tcp_component = { .exclude = NULL }; -static char *includeparam; -static char *excludeparam; - static pmix_status_t component_register(void) { pmix_mca_base_component_t *component = &mca_pnet_tcp_component.super.base; @@ -86,26 +83,26 @@ static pmix_status_t component_register(void) PMIX_MCA_BASE_VAR_SCOPE_READONLY, &mca_pnet_tcp_component.default_request); - includeparam = NULL; + mca_pnet_tcp_component.incparms = NULL; (void)pmix_mca_base_component_var_register(component, "include_envars", "Comma-delimited list of envars to harvest (\'*\' and \'?\' supported)", PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, PMIX_INFO_LVL_2, PMIX_MCA_BASE_VAR_SCOPE_LOCAL, - &includeparam); - if (NULL != includeparam) { - mca_pnet_tcp_component.include = pmix_argv_split(includeparam, ','); + &mca_pnet_tcp_component.incparms); + if (NULL != mca_pnet_tcp_component.incparms) { + mca_pnet_tcp_component.include = pmix_argv_split(mca_pnet_tcp_component.incparms, ','); } - excludeparam = NULL; + mca_pnet_tcp_component.excparms = NULL; (void)pmix_mca_base_component_var_register(component, "exclude_envars", "Comma-delimited list of envars to exclude (\'*\' and \'?\' supported)", PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, PMIX_INFO_LVL_2, PMIX_MCA_BASE_VAR_SCOPE_LOCAL, - &excludeparam); - if (NULL != excludeparam) { - mca_pnet_tcp_component.exclude = pmix_argv_split(excludeparam, ','); + &mca_pnet_tcp_component.excparms); + if (NULL != mca_pnet_tcp_component.excparms) { + mca_pnet_tcp_component.exclude = pmix_argv_split(mca_pnet_tcp_component.excparms, ','); } return PMIX_SUCCESS; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.c index a8808b43e6..830e0c02e5 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pnet/test/pnet_test.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * * $COPYRIGHT$ @@ -46,18 +46,18 @@ static pmix_status_t test_init(void); static void test_finalize(void); -static pmix_status_t allocate(pmix_nspace_t *nptr, +static pmix_status_t allocate(pmix_namespace_t *nptr, pmix_info_t *info, pmix_list_t *ilist); -static pmix_status_t setup_local_network(pmix_nspace_t *nptr, +static pmix_status_t setup_local_network(pmix_namespace_t *nptr, pmix_info_t info[], size_t ninfo); -static pmix_status_t setup_fork(pmix_nspace_t *nptr, +static pmix_status_t setup_fork(pmix_namespace_t *nptr, const pmix_proc_t *proc, char ***env); static void child_finalized(pmix_proc_t *peer); -static void local_app_finalized(pmix_nspace_t *nptr); -static void deregister_nspace(pmix_nspace_t *nptr); +static void local_app_finalized(pmix_namespace_t *nptr); +static void deregister_nspace(pmix_namespace_t *nptr); static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, pmix_inventory_cbfunc_t cbfunc, void *cbdata); static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, @@ -94,7 +94,7 @@ static void test_finalize(void) /* NOTE: if there is any binary data to be transferred, then * this function MUST pack it for transport as the host will * not know how to do so */ -static pmix_status_t allocate(pmix_nspace_t *nptr, +static pmix_status_t allocate(pmix_namespace_t *nptr, pmix_info_t *info, pmix_list_t *ilist) { @@ -283,7 +283,7 @@ static pmix_status_t allocate(pmix_nspace_t *nptr, return PMIX_SUCCESS; } -static pmix_status_t setup_local_network(pmix_nspace_t *nptr, +static pmix_status_t setup_local_network(pmix_namespace_t *nptr, pmix_info_t info[], size_t ninfo) { @@ -345,8 +345,13 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, cnt = 1; PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, &bkt, &nkvals, &cnt, PMIX_SIZE); - /* setup the info array */ - PMIX_INFO_CREATE(jinfo, nkvals); + /* the data gets stored as a pmix_data_array_t on the provided key */ + PMIX_INFO_CONSTRUCT(&stinfo); + pmix_strncpy(stinfo.key, idkey, PMIX_MAX_KEYLEN); + stinfo.value.type = PMIX_DATA_ARRAY; + PMIX_DATA_ARRAY_CREATE(stinfo.value.data.darray, nkvals, PMIX_INFO); + jinfo = (pmix_info_t*)stinfo.value.data.darray->array; + /* cycle thru the blob and extract the kvals */ kv = PMIX_NEW(pmix_kval_t); cnt = 1; @@ -358,7 +363,7 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, "recvd KEY %s %s", kv->key, (PMIX_STRING == kv->value->type) ? kv->value->data.string : "NON-STRING"); /* xfer the value to the info */ - (void)strncpy(jinfo[m].key, kv->key, PMIX_MAX_KEYLEN); + pmix_strncpy(jinfo[m].key, kv->key, PMIX_MAX_KEYLEN); PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, &jinfo[m].value, kv->value); /* if this is the ID key, save it */ @@ -384,14 +389,7 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, PMIX_INFO_FREE(jinfo, nkvals); return PMIX_ERR_BAD_PARAM; } - /* the data gets stored as a pmix_data_array_t on the provided key */ - PMIX_INFO_CONSTRUCT(&stinfo); - (void)strncpy(stinfo.key, idkey, PMIX_MAX_KEYLEN); - stinfo.value.type = PMIX_DATA_ARRAY; - PMIX_DATA_ARRAY_CREATE(stinfo.value.data.darray, nkvals, PMIX_INFO); - stinfo.value.data.darray->array = jinfo; - - /* cache the info on the job */ + /* cache the info on the job */ PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, &stinfo, 1); PMIX_INFO_DESTRUCT(&stinfo); @@ -404,7 +402,7 @@ static pmix_status_t setup_local_network(pmix_nspace_t *nptr, return PMIX_SUCCESS; } -static pmix_status_t setup_fork(pmix_nspace_t *nptr, +static pmix_status_t setup_fork(pmix_namespace_t *nptr, const pmix_proc_t *proc, char ***env) { @@ -457,12 +455,12 @@ static void child_finalized(pmix_proc_t *peer) peer->nspace, peer->rank); } -static void local_app_finalized(pmix_nspace_t *nptr) +static void local_app_finalized(pmix_namespace_t *nptr) { pmix_output(0, "pnet:test NSPACE %s LOCALLY FINALIZED", nptr->nspace); } -static void deregister_nspace(pmix_nspace_t *nptr) +static void deregister_nspace(pmix_namespace_t *nptr) { pmix_output(0, "pnet:test DEREGISTER NSPACE %s", nptr->nspace); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/preg/base/preg_base_frame.c b/opal/mca/pmix/pmix3x/pmix/src/mca/preg/base/preg_base_frame.c index dbf551ea64..706c2bc8aa 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/preg/base/preg_base_frame.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/preg/base/preg_base_frame.c @@ -14,6 +14,7 @@ * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -98,6 +99,7 @@ static void rvcon(pmix_regex_value_t *p) p->prefix = NULL; p->suffix = NULL; p->num_digits = 0; + p->skip = false; PMIX_CONSTRUCT(&p->ranges, pmix_list_t); } static void rvdes(pmix_regex_value_t *p) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/preg/native/preg_native.c b/opal/mca/pmix/pmix3x/pmix/src/mca/preg/native/preg_native.c index 1a98766f8e..0d31f96435 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/preg/native/preg_native.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/preg/native/preg_native.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2019 IBM Corporation. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * @@ -152,9 +152,22 @@ static pmix_status_t generate_node_regex(const char *input, suffix = NULL; numdigits = (int)strlen(&vptr[startnum]); } + /* is this value already on our list? */ found = false; PMIX_LIST_FOREACH(vreg, &vids, pmix_regex_value_t) { + // The regex must preserve ordering of the values. + // If we disqualified this entry in a previous check then exclude it + // from future checks as well. This will prevent a later entry from + // being 'pulled forward' accidentally. For example, given: + // "a28n01,a99n02,a28n02" + // Without this 'skip' the loop would have 'a28n02' combine with + // 'a28n01' jumping over the 'a99n02' entry, and thus not preserving + // the order of the list when the regex is unpacked. + if( vreg->skip ) { + continue; + } + if (0 < strlen(prefix) && NULL == vreg->prefix) { continue; } @@ -163,6 +176,7 @@ static pmix_status_t generate_node_regex(const char *input, } if (0 < strlen(prefix) && NULL != vreg->prefix && 0 != strcmp(prefix, vreg->prefix)) { + vreg->skip = true; continue; } if (NULL == suffix && NULL != vreg->suffix) { @@ -173,9 +187,11 @@ static pmix_status_t generate_node_regex(const char *input, } if (NULL != suffix && NULL != vreg->suffix && 0 != strcmp(suffix, vreg->suffix)) { + vreg->skip = true; continue; } if (numdigits != vreg->num_digits) { + vreg->skip = true; continue; } /* found a match - flag it */ @@ -522,7 +538,7 @@ static pmix_status_t resolve_peers(const char *nodename, /* scope is irrelevant as the info we seek must be local */ cb.scope = PMIX_SCOPE_UNDEF; /* let the proc point to the nspace */ - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; cb.proc = &proc; @@ -565,7 +581,7 @@ static pmix_status_t resolve_peers(const char *nodename, goto complete; } for (j=0; j < np; j++) { - (void)strncpy(p[j].nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(p[j].nspace, nspace, PMIX_MAX_NSLEN); p[j].rank = strtoul(ptr[j], NULL, 10); } rc = PMIX_SUCCESS; @@ -619,7 +635,7 @@ static pmix_status_t resolve_nodes(const char *nspace, /* scope is irrelevant as the info we seek must be local */ cb.scope = PMIX_SCOPE_UNDEF; /* put the nspace in the proc field */ - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); /* the info will be associated with PMIX_RANK_WILDCARD */ proc.rank = PMIX_RANK_WILDCARD; cb.proc = &proc; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/preg/preg_types.h b/opal/mca/pmix/pmix3x/pmix/src/mca/preg/preg_types.h index 9f1b8a8ae5..932d8e552c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/preg/preg_types.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/preg/preg_types.h @@ -13,6 +13,7 @@ * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2019 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -51,6 +52,7 @@ typedef struct { char *suffix; int num_digits; pmix_list_t ranges; + bool skip; } pmix_regex_value_t; PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_regex_value_t); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/psensor/file/psensor_file.c b/opal/mca/pmix/pmix3x/pmix/src/mca/psensor/file/psensor_file.c index 914e895a40..3a050823c2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/psensor/file/psensor_file.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/psensor/file/psensor_file.c @@ -345,7 +345,7 @@ static void file_sample(int sd, short args, void *cbdata) /* stop monitoring this client */ pmix_list_remove_item(&mca_psensor_file_component.trackers, &ft->super); /* generate an event */ - (void)strncpy(source.nspace, ft->requestor->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(source.nspace, ft->requestor->info->pname.nspace, PMIX_MAX_NSLEN); source.rank = ft->requestor->info->pname.rank; rc = PMIx_Notify_event(PMIX_MONITOR_FILE_ALERT, &source, ft->range, ft->info, ft->ninfo, opcbfunc, ft); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c b/opal/mca/pmix/pmix3x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c index f88ef0cdb6..81de240b65 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/psensor/heartbeat/psensor_heartbeat.c @@ -297,7 +297,7 @@ static void check_heartbeat(int fd, short dummy, void *cbdata) pmix_globals.myid.nspace, pmix_globals.myid.rank, ft->requestor->info->pname.nspace, ft->requestor->info->pname.rank)); /* generate an event */ - (void)strncpy(source.nspace, ft->requestor->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(source.nspace, ft->requestor->info->pname.nspace, PMIX_MAX_NSLEN); source.rank = ft->requestor->info->pname.rank; /* ensure the tracker remains throughout the process */ PMIX_RETAIN(ft); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pshmem/base/pshmem_base_frame.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pshmem/base/pshmem_base_frame.c index 4c38005da6..30296755f6 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pshmem/base/pshmem_base_frame.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pshmem/base/pshmem_base_frame.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -60,6 +60,9 @@ static pmix_status_t pmix_pshmem_close(void) static pmix_status_t pmix_pshmem_open(pmix_mca_base_open_flag_t flags) { + if (initialized) { + return PMIX_SUCCESS; + } /* initialize globals */ initialized = true; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/pshmem/mmap/pshmem_mmap.c b/opal/mca/pmix/pmix3x/pmix/src/mca/pshmem/mmap/pshmem_mmap.c index a004ac2731..09c377cd8b 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/pshmem/mmap/pshmem_mmap.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/pshmem/mmap/pshmem_mmap.c @@ -3,7 +3,7 @@ * All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Intel, Inc. All rights reserved. + * Copyright (c) 2017-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -121,7 +121,7 @@ static int _mmap_segment_create(pmix_pshmem_seg_t *sm_seg, const char *file_name sm_seg->seg_cpid = my_pid; sm_seg->seg_size = size; sm_seg->seg_base_addr = (unsigned char *)seg_addr; - (void)strncpy(sm_seg->seg_name, file_name, PMIX_PATH_MAX - 1); + pmix_strncpy(sm_seg->seg_name, file_name, PMIX_PATH_MAX); out: if (-1 != sm_seg->seg_id) { diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_frame.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_frame.c index 2f1fd4f6a0..2e6a101752 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_frame.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_frame.c @@ -198,9 +198,11 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_ptl_sr_t, static void pccon(pmix_pending_connection_t *p) { + p->need_id = false; memset(p->nspace, 0, PMIX_MAX_NSLEN+1); p->info = NULL; p->ninfo = 0; + p->peer = NULL; p->bfrops = NULL; p->psec = NULL; p->gds = NULL; @@ -258,6 +260,8 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_listener_t, static void qcon(pmix_ptl_queue_t *p) { p->peer = NULL; + p->buf = NULL; + p->tag = UINT32_MAX; } static void qdes(pmix_ptl_queue_t *p) { diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c index b70da16a36..0b465340be 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015-2017 Research Organization for Information Science @@ -55,16 +55,30 @@ static void _notify_complete(pmix_status_t status, void *cbdata) PMIX_RELEASE(chain); } +static void _timeout(int sd, short args, void *cbdata) +{ + pmix_server_trkr_t *trk = (pmix_server_trkr_t*)cbdata; + + PMIX_RELEASE(trk); +} + +static void lcfn(pmix_status_t status, void *cbdata) +{ + pmix_peer_t *peer = (pmix_peer_t*)cbdata; + PMIX_RELEASE(peer); +} + void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) { - pmix_server_trkr_t *trk; + pmix_server_trkr_t *trk, *tnxt; pmix_server_caddy_t *rinfo, *rnext; - pmix_regevents_info_t *reginfoptr, *regnext; - pmix_peer_events_info_t *pr, *pnext; pmix_rank_info_t *info, *pinfo; pmix_ptl_posted_recv_t *rcv; pmix_buffer_t buf; pmix_ptl_hdr_t hdr; + struct timeval tv = {1200, 0}; + pmix_proc_t proc; + pmix_status_t rc; /* stop all events */ if (peer->recv_ev_active) { @@ -82,20 +96,17 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) CLOSE_THE_SOCKET(peer->sd); if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && - !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + !PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { /* if I am a server, then we need to ensure that * we properly account for the loss of this client * from any local collectives in which it was * participating - note that the proc would not * have been added to any collective tracker until * after it successfully connected */ - PMIX_LIST_FOREACH(trk, &pmix_server_globals.collectives, pmix_server_trkr_t) { + PMIX_LIST_FOREACH_SAFE(trk, tnxt, &pmix_server_globals.collectives, pmix_server_trkr_t) { /* see if this proc is participating in this tracker */ PMIX_LIST_FOREACH_SAFE(rinfo, rnext, &trk->local_cbs, pmix_server_caddy_t) { - if (0 != strncmp(rinfo->peer->info->pname.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN)) { - continue; - } - if (rinfo->peer->info->pname.rank != peer->info->pname.rank) { + if (!PMIX_CHECK_PROCID(&rinfo->peer->info->pname, &peer->info->pname)) { continue; } /* it is - adjust the count */ @@ -103,24 +114,64 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) /* remove it from the list */ pmix_list_remove_item(&trk->local_cbs, &rinfo->super); PMIX_RELEASE(rinfo); - /* we need to let the other participants know that this - * proc has disappeared as otherwise the collective will never - * complete */ + trk->lost_connection = true; // mark that a peer's connection was lost + if (0 == pmix_list_get_size(&trk->local_cbs)) { + /* this tracker is complete, so release it - there + * is nobody waiting for a response */ + pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super); + /* do NOT release the tracker here as the host may + * have a copy they will return later. However, they + * might never call back, so set a LONG timeout to + * we avoid a memory leak if they don't */ + pmix_event_evtimer_set(pmix_globals.evbase, &trk->ev, + _timeout, trk); + pmix_event_evtimer_add(&trk->ev, &tv); + trk->event_active = true; + break; + } + /* if there are other participants waiting for a response, + * we need to let them know that this proc has disappeared + * as otherwise the collective will never complete */ if (PMIX_FENCENB_CMD == trk->type) { if (NULL != trk->modexcbfunc) { + /* do NOT release the tracker here as the host may + * have a copy they will return later. However, they + * might never call back, so set a LONG timeout to + * we avoid a memory leak if they don't */ + pmix_event_evtimer_set(pmix_globals.evbase, &trk->ev, + _timeout, trk); + pmix_event_evtimer_add(&trk->ev, &tv); + trk->event_active = true; trk->modexcbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, NULL, 0, trk, NULL, NULL); } } else if (PMIX_CONNECTNB_CMD == trk->type) { if (NULL != trk->op_cbfunc) { + /* do NOT release the tracker here as the host may + * have a copy they will return later. However, they + * might never call back, so set a LONG timeout to + * we avoid a memory leak if they don't */ + pmix_event_evtimer_set(pmix_globals.evbase, &trk->ev, + _timeout, trk); + pmix_event_evtimer_add(&trk->ev, &tv); + trk->event_active = true; trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, trk); } } else if (PMIX_DISCONNECTNB_CMD == trk->type) { if (NULL != trk->op_cbfunc) { + /* do NOT release the tracker here as the host may + * have a copy they will return later. However, they + * might never call back, so set a LONG timeout to + * we avoid a memory leak if they don't */ + pmix_event_evtimer_set(pmix_globals.evbase, &trk->ev, + _timeout, trk); + pmix_event_evtimer_add(&trk->ev, &tv); + trk->event_active = true; trk->op_cbfunc(PMIX_ERR_LOST_CONNECTION_TO_CLIENT, trk); } } } } + /* remove this proc from the list of ranks for this nspace if it is * still there - we must check for multiple copies as there will be * one for each "clone" of this peer */ @@ -130,38 +181,53 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) } } /* reduce the number of local procs */ - --peer->nptr->nlocalprocs; + if (0 < peer->nptr->nlocalprocs) { + --peer->nptr->nlocalprocs; + } /* remove this client from our array */ pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); - /* cleanup any remaining events they have registered for */ - PMIX_LIST_FOREACH_SAFE(reginfoptr, regnext, &pmix_server_globals.events, pmix_regevents_info_t) { - PMIX_LIST_FOREACH_SAFE(pr, pnext, ®infoptr->peers, pmix_peer_events_info_t) { - if (peer == pr->peer) { - pmix_list_remove_item(®infoptr->peers, &pr->super); - PMIX_RELEASE(pr); - if (0 == pmix_list_get_size(®infoptr->peers)) { - pmix_list_remove_item(&pmix_server_globals.events, ®infoptr->super); - PMIX_RELEASE(reginfoptr); - break; - } - } - } - } - /* cleanup any sensors that are monitoring them */ - pmix_psensor.stop(peer, NULL); - if (!peer->finalized && !PMIX_PROC_IS_TOOL(peer)) { + /* purge any notifications cached for this client */ + pmix_server_purge_events(peer, NULL); + + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + /* only connection I can lose is to my server, so mark it */ + pmix_globals.connected = false; + } else { + /* cleanup any sensors that are monitoring them */ + pmix_psensor.stop(peer, NULL); + } + + if (!peer->finalized && !PMIX_PROC_IS_TOOL(peer) && !pmix_globals.mypeer->finalized) { /* if this peer already called finalize, then * we are just seeing their connection go away * when they terminate - so do not generate * an event. If not, then we do */ - PMIX_REPORT_EVENT(err, peer, PMIX_RANGE_NAMESPACE, _notify_complete); + PMIX_REPORT_EVENT(err, peer, PMIX_RANGE_PROC_LOCAL, _notify_complete); } /* now decrease the refcount - might actually free the object */ PMIX_RELEASE(peer->info); + /* be sure to let the host know that the tool or client + * is gone - otherwise, it won't know to cleanup the + * resources it allocated to it */ + if (NULL != pmix_host_server.client_finalized && !peer->finalized) { + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + proc.rank = peer->info->pname.rank; + /* now tell the host server */ + rc = pmix_host_server.client_finalized(&proc, peer->info->server_object, + lcfn, peer); + if (PMIX_SUCCESS == rc) { + /* we will release the peer when the server calls us back */ + peer->finalized = true; + return; + } + } + /* mark the peer as "gone" since a release doesn't guarantee + * that the peer object doesn't persist */ + peer->finalized = true; /* Release peer info */ PMIX_RELEASE(peer); } else { @@ -191,7 +257,7 @@ void pmix_ptl_base_lost_connection(pmix_peer_t *peer, pmix_status_t err) PMIX_DESTRUCT(&buf); /* if I called finalize, then don't generate an event */ if (!pmix_globals.mypeer->finalized) { - PMIX_REPORT_EVENT(err, pmix_client_globals.myserver, PMIX_RANGE_LOCAL, _notify_complete); + PMIX_REPORT_EVENT(err, pmix_client_globals.myserver, PMIX_RANGE_PROC_LOCAL, _notify_complete); } } } @@ -343,7 +409,9 @@ void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata) if (NULL != msg) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:base:send_handler SENDING MSG"); + "ptl:base:send_handler SENDING MSG TO %s:%d TAG %u", + peer->info->pname.nspace, peer->info->pname.rank, + ntohl(msg->hdr.tag)); if (PMIX_SUCCESS == (rc = send_msg(peer->sd, msg))) { // message is complete pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -574,10 +642,10 @@ void pmix_ptl_base_send(int sd, short args, void *cbdata) if (NULL == queue->peer || queue->peer->sd < 0 || NULL == queue->peer->info || NULL == queue->peer->nptr) { /* this peer has lost connection */ + if (NULL != queue->buf) { + PMIX_RELEASE(queue->buf); + } PMIX_RELEASE(queue); - /* ensure we post the object before another thread - * picks it back up */ - PMIX_POST_OBJECT(queue); return; } @@ -587,6 +655,12 @@ void pmix_ptl_base_send(int sd, short args, void *cbdata) (queue->peer)->info->pname.nspace, (queue->peer)->info->pname.rank, (queue->tag)); + if (NULL == queue->buf) { + /* nothing to send? */ + PMIX_RELEASE(queue); + return; + } + snd = PMIX_NEW(pmix_ptl_send_t); snd->hdr.pindex = htonl(pmix_globals.pindex); snd->hdr.tag = htonl(queue->tag); @@ -623,12 +697,19 @@ void pmix_ptl_base_send_recv(int fd, short args, void *cbdata) /* acquire the object */ PMIX_ACQUIRE_OBJECT(ms); - if (ms->peer->sd < 0) { - /* this peer's socket has been closed */ + if (NULL == ms->peer || ms->peer->sd < 0 || + NULL == ms->peer->info || NULL == ms->peer->nptr) { + /* this peer has lost connection */ + if (NULL != ms->bfr) { + PMIX_RELEASE(ms->bfr); + } + PMIX_RELEASE(ms); + return; + } + + if (NULL == ms->bfr) { + /* nothing to send? */ PMIX_RELEASE(ms); - /* ensure we post the object before another thread - * picks it back up */ - PMIX_POST_OBJECT(NULL); return; } @@ -739,7 +820,9 @@ void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata) /* if the tag in this message is above the dynamic marker, then * that is an error */ if (PMIX_PTL_TAG_DYNAMIC <= msg->hdr.tag) { - pmix_output(0, "UNEXPECTED MESSAGE tag = %d", msg->hdr.tag); + pmix_output(0, "UNEXPECTED MESSAGE tag = %d from source %s:%d", + msg->hdr.tag, msg->peer->info->pname.nspace, + msg->peer->info->pname.rank); PMIX_REPORT_EVENT(PMIX_ERROR, msg->peer, PMIX_RANGE_NAMESPACE, _notify_complete); PMIX_RELEASE(msg); return; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h index 01f849b805..d413a21004 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl.h @@ -143,11 +143,23 @@ typedef struct pmix_ptl_module_t pmix_ptl_module_t; /***** MACROS FOR EXECUTING PTL FUNCTIONS *****/ -#define PMIX_PTL_SEND_RECV(r, p, b, c, d) \ - (r) = (p)->nptr->compat.ptl->send_recv((struct pmix_peer_t*)(p), b, c, d) +#define PMIX_PTL_SEND_RECV(r, p, b, c, d) \ + do { \ + if ((p)->finalized) { \ + (r) = PMIX_ERR_UNREACH; \ + } else { \ + (r) = (p)->nptr->compat.ptl->send_recv((struct pmix_peer_t*)(p), b, c, d); \ + } \ + } while(0) -#define PMIX_PTL_SEND_ONEWAY(r, p, b, t) \ - (r) = (p)->nptr->compat.ptl->send((struct pmix_peer_t*)(p), b, t) +#define PMIX_PTL_SEND_ONEWAY(r, p, b, t) \ + do { \ + if ((p)->finalized) { \ + (r) = PMIX_ERR_UNREACH; \ + } else { \ + (r) = (p)->nptr->compat.ptl->send((struct pmix_peer_t*)(p), b, t); \ + } \ + } while(0) #define PMIX_PTL_RECV(r, p, c, t) \ (r) = (p)->nptr->compat.ptl->recv((struct pmix_peer_t*)(p), c, t) diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h index 0008bb48d3..0017c5b813 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/ptl_types.h @@ -63,15 +63,16 @@ struct pmix_ptl_module_t; /* define a process type */ typedef uint16_t pmix_proc_type_t; + #define PMIX_PROC_UNDEF 0x0000 -#define PMIX_PROC_CLIENT 0x0001 -#define PMIX_PROC_SERVER 0x0002 -#define PMIX_PROC_TOOL 0x0004 -#define PMIX_PROC_V1 0x0008 -#define PMIX_PROC_V20 0x0010 -#define PMIX_PROC_V21 0x0020 -#define PMIX_PROC_V3 0x0040 -#define PMIX_PROC_LAUNCHER_ACT 0x1000 +#define PMIX_PROC_CLIENT 0x0001 // simple client process +#define PMIX_PROC_SERVER 0x0002 // simple server process +#define PMIX_PROC_TOOL 0x0004 // simple tool +#define PMIX_PROC_V1 0x0008 // process is using PMIx v1 protocols +#define PMIX_PROC_V20 0x0010 // process is using PMIx v2.0 protocols +#define PMIX_PROC_V21 0x0020 // process is using PMIx v2.1 protocols +#define PMIX_PROC_V3 0x0040 // process is using PMIx v3 protocols +#define PMIX_PROC_LAUNCHER_ACT 0x1000 // process acting as launcher #define PMIX_PROC_LAUNCHER (PMIX_PROC_TOOL | PMIX_PROC_SERVER | PMIX_PROC_LAUNCHER_ACT) #define PMIX_PROC_CLIENT_TOOL_ACT 0x2000 #define PMIX_PROC_CLIENT_TOOL (PMIX_PROC_TOOL | PMIX_PROC_CLIENT | PMIX_PROC_CLIENT_TOOL_ACT) @@ -196,11 +197,14 @@ typedef struct { pmix_event_t ev; pmix_listener_protocol_t protocol; int sd; + bool need_id; + uint8_t flag; char nspace[PMIX_MAX_NSLEN+1]; pmix_info_t *info; size_t ninfo; pmix_status_t status; struct sockaddr_storage addr; + struct pmix_peer_t *peer; char *bfrops; char *psec; char *gds; @@ -236,9 +240,6 @@ PMIX_EXPORT extern int pmix_ptl_base_output; #define PMIX_ACTIVATE_POST_MSG(ms) \ do { \ - pmix_output_verbose(5, pmix_ptl_base_output, \ - "[%s:%d] post msg", \ - __FILE__, __LINE__); \ pmix_event_assign(&((ms)->ev), pmix_globals.evbase, -1, \ EV_WRITE, pmix_ptl_base_process_msg, (ms)); \ PMIX_POST_OBJECT(ms); \ @@ -259,37 +260,42 @@ PMIX_EXPORT extern int pmix_ptl_base_output; * t - tag to be sent to * b - buffer to be sent */ -#define PMIX_SERVER_QUEUE_REPLY(p, t, b) \ - do { \ - pmix_ptl_send_t *snd; \ - uint32_t nbytes; \ - pmix_output_verbose(5, pmix_ptl_base_output, \ +#define PMIX_SERVER_QUEUE_REPLY(r, p, t, b) \ + do { \ + pmix_ptl_send_t *snd; \ + uint32_t nbytes; \ + pmix_output_verbose(5, pmix_ptl_base_output, \ "[%s:%d] queue callback called: reply to %s:%d on tag %d size %d", \ - __FILE__, __LINE__, \ - (p)->info->pname.nspace, \ - (p)->info->pname.rank, (t), (int)(b)->bytes_used); \ - snd = PMIX_NEW(pmix_ptl_send_t); \ - snd->hdr.pindex = htonl(pmix_globals.pindex); \ - snd->hdr.tag = htonl(t); \ - nbytes = (b)->bytes_used; \ - snd->hdr.nbytes = htonl(nbytes); \ - snd->data = (b); \ - /* always start with the header */ \ - snd->sdptr = (char*)&snd->hdr; \ - snd->sdbytes = sizeof(pmix_ptl_hdr_t); \ - /* if there is no message on-deck, put this one there */ \ - if (NULL == (p)->send_msg) { \ - (p)->send_msg = snd; \ - } else { \ - /* add it to the queue */ \ - pmix_list_append(&(p)->send_queue, &snd->super); \ - } \ - /* ensure the send event is active */ \ - if (!(p)->send_ev_active && 0 <= (p)->sd) { \ - (p)->send_ev_active = true; \ - PMIX_POST_OBJECT(snd); \ - pmix_event_add(&(p)->send_event, 0); \ - } \ + __FILE__, __LINE__, \ + (p)->info->pname.nspace, \ + (p)->info->pname.rank, (t), (int)(b)->bytes_used); \ + if ((p)->finalized) { \ + (r) = PMIX_ERR_UNREACH; \ + } else { \ + snd = PMIX_NEW(pmix_ptl_send_t); \ + snd->hdr.pindex = htonl(pmix_globals.pindex); \ + snd->hdr.tag = htonl(t); \ + nbytes = (b)->bytes_used; \ + snd->hdr.nbytes = htonl(nbytes); \ + snd->data = (b); \ + /* always start with the header */ \ + snd->sdptr = (char*)&snd->hdr; \ + snd->sdbytes = sizeof(pmix_ptl_hdr_t); \ + /* if there is no message on-deck, put this one there */ \ + if (NULL == (p)->send_msg) { \ + (p)->send_msg = snd; \ + } else { \ + /* add it to the queue */ \ + pmix_list_append(&(p)->send_queue, &snd->super); \ + } \ + /* ensure the send event is active */ \ + if (!(p)->send_ev_active && 0 <= (p)->sd) { \ + (p)->send_ev_active = true; \ + PMIX_POST_OBJECT(snd); \ + pmix_event_add(&(p)->send_event, 0); \ + } \ + (r) = PMIX_SUCCESS; \ + } \ } while (0) #define CLOSE_THE_SOCKET(s) \ diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c index 43fb426acb..e921cd599c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.c @@ -13,7 +13,8 @@ * Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -42,7 +43,12 @@ #ifdef HAVE_SYS_STAT_H #include #endif +#ifdef HAVE_DIRENT_H #include +#endif +#ifdef HAVE_SYS_SYSCTL_H +#include +#endif #include "src/include/pmix_globals.h" #include "src/include/pmix_socket_errno.h" @@ -53,6 +59,7 @@ #include "src/util/os_path.h" #include "src/util/show_help.h" #include "src/mca/bfrops/base/base.h" +#include "src/mca/gds/gds.h" #include "src/mca/ptl/base/base.h" #include "ptl_tcp.h" @@ -77,8 +84,8 @@ pmix_ptl_module_t pmix_ptl_tcp_module = { .connect_to_peer = connect_to_peer }; -static pmix_status_t recv_connect_ack(int sd); -static pmix_status_t send_connect_ack(int sd); +static pmix_status_t recv_connect_ack(int sd, uint8_t myflag); +static pmix_status_t send_connect_ack(int sd, uint8_t *myflag, pmix_info_t info[], size_t ninfo); static pmix_status_t init(void) @@ -109,10 +116,11 @@ static pmix_status_t parse_uri_file(char *filename, char **uri, char **nspace, pmix_rank_t *rank); -static pmix_status_t try_connect(char *uri, int *sd); +static pmix_status_t try_connect(char *uri, int *sd, pmix_info_t info[], size_t ninfo); static pmix_status_t df_search(char *dirname, char *prefix, + pmix_info_t info[], size_t ninfo, int *sd, char **nspace, - pmix_rank_t *rank); + pmix_rank_t *rank, char **uri); static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_info_t *info, size_t ninfo) @@ -120,14 +128,19 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, char *evar, **uri, *suri = NULL, *suri2 = NULL; char *filename, *nspace=NULL; pmix_rank_t rank = PMIX_RANK_WILDCARD; - char *p, *p2, *server_nspace = NULL; + char *p, *p2, *server_nspace = NULL, *rendfile = NULL; int sd, rc; size_t n; char myhost[PMIX_MAXHOSTNAMELEN]; bool system_level = false; bool system_level_only = false; bool reconnect = false; - pid_t pid = 0; + pid_t pid = 0, mypid; + pmix_list_t ilist; + pmix_info_caddy_t *kv; + pmix_info_t *iptr = NULL, mypidinfo, mycmdlineinfo, launcher; + size_t niptr = 0; + pmix_kval_t *urikv = NULL; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp: connecting to server"); @@ -200,14 +213,16 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, ++p2; nspace = strdup(p); rank = strtoull(p2, NULL, 10); + suri = strdup(uri[1]); pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp:client attempt connect to %s", uri[1]); /* go ahead and try to connect */ - if (PMIX_SUCCESS != (rc = try_connect(uri[1], &sd))) { + if (PMIX_SUCCESS != (rc = try_connect(uri[1], &sd, info, ninfo))) { free(nspace); pmix_argv_free(uri); + free(suri); return rc; } pmix_argv_free(uri); @@ -218,16 +233,17 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, /* get here if we are a tool - check any provided directives * to see where they want us to connect to */ suri = NULL; + PMIX_CONSTRUCT(&ilist, pmix_list_t); if (NULL != info) { for (n=0; n < ninfo; n++) { - if (0 == strcmp(info[n].key, PMIX_CONNECT_TO_SYSTEM)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_TO_SYSTEM)) { system_level_only = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strncmp(info[n].key, PMIX_CONNECT_SYSTEM_FIRST, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_SYSTEM_FIRST)) { /* try the system-level */ system_level = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strncmp(info[n].key, PMIX_SERVER_PIDINFO, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_PIDINFO)) { pid = info[n].value.data.pid; - } else if (0 == strncmp(info[n].key, PMIX_SERVER_NSPACE, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_NSPACE)) { if (NULL != server_nspace) { /* they included it more than once */ if (0 == strcmp(server_nspace, info[n].value.data.string)) { @@ -239,10 +255,13 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, if (NULL != suri) { free(suri); } + if (NULL != rendfile) { + free(rendfile); + } return PMIX_ERR_BAD_PARAM; } server_nspace = strdup(info[n].value.data.string); - } else if (0 == strncmp(info[n].key, PMIX_SERVER_URI, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_URI)) { if (NULL != suri) { /* they included it more than once */ if (0 == strcmp(suri, info[n].value.data.string)) { @@ -254,18 +273,141 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, if (NULL != server_nspace) { free(server_nspace); } + if (NULL != rendfile) { + free(rendfile); + } return PMIX_ERR_BAD_PARAM; } suri = strdup(info[n].value.data.string); - } else if (0 == strncmp(info[n].key, PMIX_CONNECT_RETRY_DELAY, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_RETRY_DELAY)) { mca_ptl_tcp_component.wait_to_connect = info[n].value.data.uint32; - } else if (0 == strncmp(info[n].key, PMIX_CONNECT_MAX_RETRIES, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_CONNECT_MAX_RETRIES)) { mca_ptl_tcp_component.max_retries = info[n].value.data.uint32; - } else if (0 == strncmp(info[n].key, PMIX_RECONNECT_SERVER, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_RECONNECT_SERVER)) { reconnect = true; + } else if (PMIX_CHECK_KEY(&info[n], PMIX_LAUNCHER_RENDEZVOUS_FILE)) { + if (NULL != rendfile) { + free(rendfile); + } + rendfile = strdup(info[n].value.data.string); + } else { + /* need to pass this to server */ + kv = PMIX_NEW(pmix_info_caddy_t); + kv->info = &info[n]; + pmix_list_append(&ilist, &kv->super); } } } + /* add our pid to the array */ + kv = PMIX_NEW(pmix_info_caddy_t); + mypid = getpid(); + PMIX_INFO_LOAD(&mypidinfo, PMIX_PROC_PID, &mypid, PMIX_PID); + kv->info = &mypidinfo; + pmix_list_append(&ilist, &kv->super); + + /* if I am a launcher, tell them so */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + kv = PMIX_NEW(pmix_info_caddy_t); + PMIX_INFO_LOAD(&launcher, PMIX_LAUNCHER, NULL, PMIX_BOOL); + kv->info = &launcher; + pmix_list_append(&ilist, &kv->super); + } + + /* add our cmd line to the array */ +#if PMIX_HAVE_APPLE + int mib[3], argmax, nargs, num; + size_t size; + char *procargs, *cp, *cptr; + char **stack = NULL; + + /* Get the maximum process arguments size. */ + mib[0] = CTL_KERN; + mib[1] = KERN_ARGMAX; + size = sizeof(argmax); + + if (sysctl(mib, 2, &argmax, &size, NULL, 0) == -1) { + fprintf(stderr, "sysctl() argmax failed\n"); + return -1; + } + + /* Allocate space for the arguments. */ + procargs = (char *)malloc(argmax); + if (procargs == NULL) + return -1; + + /* Make a sysctl() call to get the raw argument space of the process. */ + mib[0] = CTL_KERN; + mib[1] = KERN_PROCARGS2; + mib[2] = getpid(); + + size = (size_t)argmax; + + if (sysctl(mib, 3, procargs, &size, NULL, 0) == -1) { + fprintf(stderr, "Lacked permissions\n");; + return 0; + } + + memcpy(&nargs, procargs, sizeof(nargs)); + /* this points to the executable - skip over that to get the rest */ + cp = procargs + sizeof(nargs); + cp += strlen(cp); + /* this is the first argv */ + pmix_argv_append_nosize(&stack, cp); + /* skip any embedded NULLs */ + while (cp < &procargs[size] && '\0' == *cp) { + ++cp; + } + if (cp != &procargs[size]) { + /* from this point, we have the argv separated by NULLs - split them out */ + cptr = cp; + num = 0; + while (cp < &procargs[size] && num < nargs) { + if ('\0' == *cp) { + pmix_argv_append_nosize(&stack, cptr); + ++cp; // skip over the NULL + cptr = cp; + ++num; + } else { + ++cp; + } + } + } + p = pmix_argv_join(stack, ' '); + pmix_argv_free(stack); + free(procargs); +#else + char tmp[512]; + FILE *fp; + + /* open the pid's info file */ + snprintf(tmp, 512, "/proc/%lu/cmdline", (unsigned long)mypid); + fp = fopen(tmp, "r"); + if (NULL != fp) { + /* read the cmd line */ + fgets(tmp, 512, fp); + fclose(fp); + p = strdup(tmp); + } +#endif + /* pass it along */ + kv = PMIX_NEW(pmix_info_caddy_t); + PMIX_INFO_LOAD(&mycmdlineinfo, PMIX_CMD_LINE, p, PMIX_STRING); + kv->info = &mycmdlineinfo; + pmix_list_append(&ilist, &kv->super); + free(p); + + /* if we need to pass anything, setup an array */ + if (0 < (niptr = pmix_list_get_size(&ilist))) { + PMIX_INFO_CREATE(iptr, niptr); + n = 0; + while (NULL != (kv = (pmix_info_caddy_t*)pmix_list_remove_first(&ilist))) { + PMIX_INFO_XFER(&iptr[n], kv->info); + PMIX_RELEASE(kv); + ++n; + } + } + PMIX_LIST_DESTRUCT(&ilist); + if (NULL == suri && !reconnect && NULL != mca_ptl_tcp_component.super.uri) { suri = strdup(mca_ptl_tcp_component.super.uri); } @@ -288,6 +430,12 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, rc = parse_uri_file(&suri[5], &suri2, &nspace, &rank); if (PMIX_SUCCESS != rc) { free(suri); + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_UNREACH; } free(suri); @@ -297,6 +445,12 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, p = strchr(suri, ';'); if (NULL == p) { free(suri); + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_BAD_PARAM; } *p = '\0'; @@ -308,6 +462,12 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, if (NULL == p) { free(suri2); free(suri); + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_BAD_PARAM; } *p = '\0'; @@ -321,74 +481,74 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp:tool attempt connect using given URI %s", suri); /* go ahead and try to connect */ - if (PMIX_SUCCESS != (rc = try_connect(suri, &sd))) { + if (PMIX_SUCCESS != (rc = try_connect(suri, &sd, iptr, niptr))) { if (NULL != nspace) { free(nspace); } free(suri); + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return rc; } + /* cleanup */ free(suri); suri = NULL; + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } goto complete; } - /* if they gave us a pid, then look for it */ - if (0 != pid) { - if (NULL != server_nspace) { - free(server_nspace); - server_nspace = NULL; - } - if (0 > asprintf(&filename, "pmix.%s.tool.%d", myhost, pid)) { - return PMIX_ERR_NOMEM; - } - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool searching for given session server %s", - filename); - nspace = NULL; - rc = df_search(mca_ptl_tcp_component.system_tmpdir, - filename, &sd, &nspace, &rank); - free(filename); + /* if they gave us a rendezvous file, use it */ + if (NULL != rendfile) { + /* try to read the file */ + rc = parse_uri_file(rendfile, &suri, &nspace, &rank); + free(rendfile); + rendfile = NULL; if (PMIX_SUCCESS == rc) { - goto complete; + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp:tool attempt connect to system server at %s", suri); + /* go ahead and try to connect */ + if (PMIX_SUCCESS == try_connect(suri, &sd, iptr, niptr)) { + /* don't free nspace - we will use it below */ + if (NULL != rendfile) { + free(rendfile); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + goto complete; + } } + /* cleanup */ if (NULL != nspace) { free(nspace); } - /* since they gave us a specific pid and we couldn't + if (NULL != suri) { + free(suri); + } + free(rendfile); + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + /* since they gave us a specific rendfile and we couldn't * connect to it, return an error */ return PMIX_ERR_UNREACH; } - /* if they gave us an nspace, then look for it */ - if (NULL != server_nspace) { - if (0 > asprintf(&filename, "pmix.%s.tool.%s", myhost, server_nspace)) { - free(server_nspace); - return PMIX_ERR_NOMEM; - } - free(server_nspace); - server_nspace = NULL; - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "ptl:tcp:tool searching for given session server %s", - filename); - nspace = NULL; - rc = df_search(mca_ptl_tcp_component.system_tmpdir, - filename, &sd, &nspace, &rank); - free(filename); - if (PMIX_SUCCESS == rc) { - goto complete; - } - if (NULL != nspace) { - free(nspace); - } - /* since they gave us a specific nspace and we couldn't - * connect to it, return an error */ - return PMIX_ERR_UNREACH; - } - - /* if they asked for system-level, we start there */ + /* if they asked for system-level first or only, we start there */ if (system_level || system_level_only) { if (0 > asprintf(&filename, "%s/pmix.sys.%s", mca_ptl_tcp_component.system_tmpdir, myhost)) { + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_NOMEM; } pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -401,8 +561,11 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp:tool attempt connect to system server at %s", suri); /* go ahead and try to connect */ - if (PMIX_SUCCESS == try_connect(suri, &sd)) { + if (PMIX_SUCCESS == try_connect(suri, &sd, iptr, niptr)) { /* don't free nspace - we will use it below */ + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } goto complete; } free(nspace); @@ -418,6 +581,80 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, if (NULL != suri) { free(suri); } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + return PMIX_ERR_UNREACH; + } + + /* if they gave us a pid, then look for it */ + if (0 != pid) { + if (NULL != server_nspace) { + free(server_nspace); + server_nspace = NULL; + } + if (0 > asprintf(&filename, "pmix.%s.tool.%d", myhost, pid)) { + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + return PMIX_ERR_NOMEM; + } + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp:tool searching for given session server %s", + filename); + nspace = NULL; + rc = df_search(mca_ptl_tcp_component.system_tmpdir, + filename, iptr, niptr, &sd, &nspace, &rank, &suri); + free(filename); + if (PMIX_SUCCESS == rc) { + goto complete; + } + if (NULL != suri) { + free(suri); + } + if (NULL != nspace) { + free(nspace); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + /* since they gave us a specific pid and we couldn't + * connect to it, return an error */ + return PMIX_ERR_UNREACH; + } + + /* if they gave us an nspace, then look for it */ + if (NULL != server_nspace) { + if (0 > asprintf(&filename, "pmix.%s.tool.%s", myhost, server_nspace)) { + free(server_nspace); + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + return PMIX_ERR_NOMEM; + } + free(server_nspace); + server_nspace = NULL; + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp:tool searching for given session server %s", + filename); + nspace = NULL; + rc = df_search(mca_ptl_tcp_component.system_tmpdir, + filename, iptr, niptr, &sd, &nspace, &rank, &suri); + free(filename); + if (PMIX_SUCCESS == rc) { + goto complete; + } + if (NULL != suri) { + free(suri); + } + if (NULL != nspace) { + free(nspace); + } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } + /* since they gave us a specific nspace and we couldn't + * connect to it, return an error */ return PMIX_ERR_UNREACH; } @@ -430,6 +667,9 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, if (NULL != suri) { free(suri); } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_NOMEM; } pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -437,7 +677,7 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, filename); nspace = NULL; rc = df_search(mca_ptl_tcp_component.system_tmpdir, - filename, &sd, &nspace, &rank); + filename, iptr, niptr, &sd, &nspace, &rank, &suri); free(filename); if (PMIX_SUCCESS != rc) { if (NULL != nspace){ @@ -446,12 +686,18 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, if (NULL != suri) { free(suri); } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } return PMIX_ERR_UNREACH; } + if (NULL != iptr) { + PMIX_INFO_FREE(iptr, niptr); + } complete: pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "sock_peer_try_connect: Connection across to server succeeded"); + "tcp_peer_try_connect: Connection across to server succeeded"); /* do a final bozo check */ if (NULL == nspace || PMIX_RANK_WILDCARD == rank) { @@ -476,7 +722,7 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); } if (NULL == pmix_client_globals.myserver->nptr) { - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); } if (NULL != pmix_client_globals.myserver->nptr->nspace) { free(pmix_client_globals.myserver->nptr->nspace); @@ -489,6 +735,16 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_client_globals.myserver->info->pname.nspace = strdup(pmix_client_globals.myserver->nptr->nspace); pmix_client_globals.myserver->info->pname.rank = rank; } + /* store the URI for subsequent lookups */ + urikv = PMIX_NEW(pmix_kval_t); + urikv->key = strdup(PMIX_SERVER_URI); + PMIX_VALUE_CREATE(urikv->value, 1); + urikv->value->type = PMIX_STRING; + asprintf(&urikv->value->data.string, "%s.%u;%s", nspace, rank, suri); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, PMIX_INTERNAL, + urikv); + PMIX_RELEASE(urikv); // maintain accounting pmix_ptl_base_set_nonblocking(sd); @@ -680,14 +936,15 @@ static pmix_status_t parse_uri_file(char *filename, return PMIX_SUCCESS; } -static pmix_status_t try_connect(char *uri, int *sd) +static pmix_status_t try_connect(char *uri, int *sd, pmix_info_t iptr[], size_t niptr) { char *p, *p2, *host; struct sockaddr_in *in; struct sockaddr_in6 *in6; size_t len; pmix_status_t rc; - bool retried = false; + int retries = 0; + uint8_t myflag; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix:tcp try connect to %s", uri); @@ -771,29 +1028,28 @@ static pmix_status_t try_connect(char *uri, int *sd) } /* send our identity and any authentication credentials to the server */ - if (PMIX_SUCCESS != (rc = send_connect_ack(*sd))) { + if (PMIX_SUCCESS != (rc = send_connect_ack(*sd, &myflag, iptr, niptr))) { PMIX_ERROR_LOG(rc); CLOSE_THE_SOCKET(*sd); return rc; } /* do whatever handshake is required */ - if (PMIX_SUCCESS != (rc = recv_connect_ack(*sd))) { + if (PMIX_SUCCESS != (rc = recv_connect_ack(*sd, myflag))) { CLOSE_THE_SOCKET(*sd); if (PMIX_ERR_TEMP_UNAVAILABLE == rc) { - /* give it two tries */ - if (!retried) { - retried = true; + ++retries; + if( retries < mca_ptl_tcp_component.handshake_max_retries ) { goto retry; } } - PMIX_ERROR_LOG(rc); return rc; } return PMIX_SUCCESS; } -static pmix_status_t send_connect_ack(int sd) +static pmix_status_t send_connect_ack(int sd, uint8_t *myflag, + pmix_info_t iptr[], size_t niptr) { char *msg; pmix_ptl_hdr_t hdr; @@ -806,7 +1062,7 @@ static pmix_status_t send_connect_ack(int sd) uid_t euid; gid_t egid; uint32_t u32; - bool self_defined = false; + pmix_buffer_t buf; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix:tcp SEND CONNECT ACK"); @@ -814,6 +1070,7 @@ static pmix_status_t send_connect_ack(int sd) /* if we are a server, then we shouldn't be here */ if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) && !PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); return PMIX_ERR_NOT_SUPPORTED; } @@ -837,35 +1094,68 @@ static pmix_status_t send_connect_ack(int sd) /* allow space for a marker indicating client vs tool */ sdsize = 1; - if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* Defined marker values: + * + * 0 => simple client process + * 1 => legacy tool - may or may not have an identifier + * 2 => legacy launcher - may or may not have an identifier + * ------------------------------------------ + * 3 => self-started tool process that needs an identifier + * 4 => self-started tool process that was given an identifier by caller + * 5 => tool that was started by a PMIx server - identifier specified by server + * 6 => self-started launcher that needs an identifier + * 7 => self-started launcher that was given an identifier by caller + * 8 => launcher that was started by a PMIx server - identifier specified by server + */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* if we are both launcher and client, then we need + * to tell the server we are both */ + flag = 8; + /* add space for our uid/gid for ACL purposes */ + sdsize += 2*sizeof(uint32_t); + /* add space for our identifier */ + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + } else { + /* add space for our uid/gid for ACL purposes */ + sdsize += 2*sizeof(uint32_t); + /* if they gave us an identifier, we need to pass it */ + if (0 < strlen(pmix_globals.myid.nspace) && + PMIX_RANK_INVALID != pmix_globals.myid.rank) { + flag = 7; + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + } else { + flag = 6; + } + } + + } else if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer) && + !PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { + /* we are a simple client */ flag = 0; /* reserve space for our nspace and rank info */ sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); - } else if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { - flag = 2; + + } else { // must be a tool of some sort /* add space for our uid/gid for ACL purposes */ sdsize += 2*sizeof(uint32_t); - /* if we already have an identifier, we need to pass it */ - if (0 < strlen(pmix_globals.myid.nspace) && + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* if we are both tool and client, then we need + * to tell the server we are both */ + flag = 5; + /* add space for our identifier */ + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + } else if (0 < strlen(pmix_globals.myid.nspace) && PMIX_RANK_INVALID != pmix_globals.myid.rank) { - sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t) + 1; - self_defined = true; + /* we were given an identifier by the caller, pass it */ + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); + flag = 4; } else { - ++sdsize; // need space for the flag indicating if have id - } - } else { // must be a simple tool - flag = 1; - /* add space for our uid/gid for ACL purposes */ - sdsize += 2*sizeof(uint32_t); - /* if we self-defined an identifier, we need to pass it */ - if (0 < strlen(pmix_globals.myid.nspace) && - PMIX_RANK_INVALID != pmix_globals.myid.rank) { - sdsize += 1 + strlen(pmix_globals.myid.nspace) + 1 + sizeof(uint32_t); - self_defined = true; - } else { - ++sdsize; // need space for the flag indicating if have id + /* we are a self-started tool that needs an identifier */ + flag = 3; } } + *myflag = flag; /* add the name of our active sec module - we selected it * in pmix_client.c prior to entering here */ @@ -879,16 +1169,26 @@ static pmix_status_t send_connect_ack(int sd) /* add our active gds module for working with the server */ gds = (char*)pmix_client_globals.myserver->nptr->compat.gds->name; - /* set the number of bytes to be read beyond the header */ + /* if we were given info structs to pass to the server, pack them */ + PMIX_CONSTRUCT(&buf, pmix_buffer_t); + if (NULL != iptr) { + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, &niptr, 1, PMIX_SIZE); + PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, iptr, niptr, PMIX_INFO); + } + + /* set the number of bytes to be read beyond the header - must + * NULL terminate the strings! */ hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 \ + strlen(bfrops) + 1 + sizeof(bftype) \ - + strlen(gds) + 1 + sizeof(uint32_t) + cred.size; // must NULL terminate the strings! + + strlen(gds) + 1 + sizeof(uint32_t) + cred.size \ + + buf.bytes_used; /* create a space for our message */ sdsize = (sizeof(hdr) + hdr.nbytes); if (NULL == (msg = (char*)malloc(sdsize))) { PMIX_BYTE_OBJECT_DESTRUCT(&cred); free(sec); + PMIX_DESTRUCT(&buf); return PMIX_ERR_OUT_OF_RESOURCE; } memset(msg, 0, sdsize); @@ -920,7 +1220,7 @@ static pmix_status_t send_connect_ack(int sd) memcpy(msg+csize, &flag, 1); csize += 1; - if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + if (0 == flag) { /* if we are a client, provide our nspace/rank */ memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); csize += strlen(pmix_globals.myid.nspace)+1; @@ -928,9 +1228,8 @@ static pmix_status_t send_connect_ack(int sd) u32 = htonl((uint32_t)pmix_globals.myid.rank); memcpy(msg+csize, &u32, sizeof(uint32_t)); csize += sizeof(uint32_t); - } else { - /* if we are a tool, provide our uid/gid for ACL support - note - * that we have to convert so we can handle heterogeneity */ + } else if (3 == flag || 6 == flag) { + /* we are a tool or launcher that needs an identifier - add our ACLs */ euid = geteuid(); u32 = htonl(euid); memcpy(msg+csize, &u32, sizeof(uint32_t)); @@ -939,6 +1238,27 @@ static pmix_status_t send_connect_ack(int sd) u32 = htonl(egid); memcpy(msg+csize, &u32, sizeof(uint32_t)); csize += sizeof(uint32_t); + } else if (4 == flag || 5 == flag || 7 == flag || 8 == flag) { + /* we are a tool or launcher that has an identifier - start with our ACLs */ + euid = geteuid(); + u32 = htonl(euid); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + egid = getegid(); + u32 = htonl(egid); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + /* now add our identifier */ + memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); + csize += strlen(pmix_globals.myid.nspace)+1; + /* again, need to convert */ + u32 = htonl((uint32_t)pmix_globals.myid.rank); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + } else { + /* not a valid flag */ + PMIX_DESTRUCT(&buf); + return PMIX_ERR_NOT_SUPPORTED; } /* provide our version */ @@ -957,46 +1277,33 @@ static pmix_status_t send_connect_ack(int sd) memcpy(msg+csize, gds, strlen(gds)); csize += strlen(gds)+1; - /* if we are not a client and self-defined an identifier, we need to pass it */ - if (!PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { - if (self_defined) { - flag = 1; - memcpy(msg+csize, &flag, 1); - ++csize; - memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); - csize += strlen(pmix_globals.myid.nspace)+1; - /* again, need to convert */ - u32 = htonl((uint32_t)pmix_globals.myid.rank); - memcpy(msg+csize, &u32, sizeof(uint32_t)); - csize += sizeof(uint32_t); - } else { - flag = 0; - memcpy(msg+csize, &flag, 1); - ++csize; - } - } + /* provide the info struct bytes */ + memcpy(msg+csize, buf.base_ptr, buf.bytes_used); + csize += buf.bytes_used; /* send the entire message across */ if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) { free(msg); + PMIX_DESTRUCT(&buf); return PMIX_ERR_UNREACH; } free(msg); + PMIX_DESTRUCT(&buf); return PMIX_SUCCESS; } /* we receive a connection acknowledgement from the server, * consisting of nothing more than a status report. If success, * then we initiate authentication method */ -static pmix_status_t recv_connect_ack(int sd) +static pmix_status_t recv_connect_ack(int sd, uint8_t myflag) { pmix_status_t reply; pmix_status_t rc; struct timeval tv, save; pmix_socklen_t sz; bool sockopt = true; + pmix_nspace_t nspace; uint32_t u32; - char nspace[PMIX_MAX_NSLEN+1]; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix: RECV CONNECT ACK FROM SERVER"); @@ -1011,12 +1318,16 @@ static pmix_status_t recv_connect_ack(int sd) } } else { /* set a timeout on the blocking recv so we don't hang */ - tv.tv_sec = 2; + tv.tv_sec = mca_ptl_tcp_component.handshake_wait_time; tv.tv_usec = 0; if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) { - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); - return PMIX_ERR_UNREACH; + if (ENOPROTOOPT == errno || EOPNOTSUPP == errno) { + sockopt = false; + } else { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); + return PMIX_ERR_UNREACH; + } } } @@ -1033,7 +1344,7 @@ static pmix_status_t recv_connect_ack(int sd) } reply = ntohl(u32); - if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + if (0 == myflag) { /* see if they want us to do the handshake */ if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { PMIX_PSEC_CLIENT_HANDSHAKE(rc, pmix_client_globals.myserver, sd); @@ -1055,26 +1366,23 @@ static pmix_status_t recv_connect_ack(int sd) } else { // we are a tool /* if the status indicates an error, then we are done */ if (PMIX_SUCCESS != reply) { - PMIX_ERROR_LOG(reply); return reply; } - /* recv our nspace */ - rc = pmix_ptl_base_recv_blocking(sd, nspace, PMIX_MAX_NSLEN+1); - if (PMIX_SUCCESS != rc) { - return rc; - } - /* if we already have our nspace, then just verify it matches */ - if (0 < strlen(pmix_globals.myid.nspace)) { - if (0 != strncmp(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN)) { - return PMIX_ERR_INIT; + /* if we needed an identifier, recv it */ + if (3 == myflag || 6 == myflag) { + /* first the nspace */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&nspace, PMIX_MAX_NSLEN+1); + if (PMIX_SUCCESS != rc) { + return rc; } - } else { - (void)strncpy(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN); - } - /* if we already have a rank, then leave it alone */ - if (PMIX_RANK_INVALID == pmix_globals.myid.rank) { - /* our rank is always zero */ - pmix_globals.myid.rank = 0; + PMIX_LOAD_NSPACE(pmix_globals.myid.nspace, nspace); + /* now the rank */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t)); + if (PMIX_SUCCESS != rc) { + return rc; + } + /* convert and store */ + pmix_globals.myid.rank = htonl(u32); } /* get the server's nspace and rank so we can send to it */ @@ -1082,7 +1390,7 @@ static pmix_status_t recv_connect_ack(int sd) pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); } if (NULL == pmix_client_globals.myserver->nptr) { - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); } pmix_ptl_base_recv_blocking(sd, (char*)nspace, PMIX_MAX_NSLEN+1); if (NULL != pmix_client_globals.myserver->nptr->nspace) { @@ -1093,7 +1401,8 @@ static pmix_status_t recv_connect_ack(int sd) free(pmix_client_globals.myserver->info->pname.nspace); } pmix_client_globals.myserver->info->pname.nspace = strdup(nspace); - pmix_ptl_base_recv_blocking(sd, (char*)&(pmix_client_globals.myserver->info->pname.rank), sizeof(int)); + pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t)); + pmix_client_globals.myserver->info->pname.rank = htonl(u32); pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix: RECV CONNECT CONFIRMATION FOR TOOL %s:%d FROM SERVER %s:%d", @@ -1127,8 +1436,9 @@ static pmix_status_t recv_connect_ack(int sd) } static pmix_status_t df_search(char *dirname, char *prefix, + pmix_info_t info[], size_t ninfo, int *sd, char **nspace, - pmix_rank_t *rank) + pmix_rank_t *rank, char **uri) { char *suri, *nsp, *newdir; pmix_rank_t rk; @@ -1158,7 +1468,7 @@ static pmix_status_t df_search(char *dirname, char *prefix, } /* if it is a directory, down search */ if (S_ISDIR(buf.st_mode)) { - rc = df_search(newdir, prefix, sd, nspace, rank); + rc = df_search(newdir, prefix, info, ninfo, sd, nspace, rank, uri); free(newdir); if (PMIX_SUCCESS == rc) { closedir(cur_dirp); @@ -1178,11 +1488,11 @@ static pmix_status_t df_search(char *dirname, char *prefix, /* go ahead and try to connect */ pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "pmix:tcp: attempting to connect to %s", suri); - if (PMIX_SUCCESS == try_connect(suri, sd)) { + if (PMIX_SUCCESS == try_connect(suri, sd, info, ninfo)) { (*nspace) = nsp; *rank = rk; closedir(cur_dirp); - free(suri); + *uri = suri; free(newdir); return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h index dd92a89381..f5373f6506 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp.h @@ -10,6 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -48,10 +49,13 @@ typedef struct { char *session_filename; char *nspace_filename; char *system_filename; + char *rendezvous_filename; int wait_to_connect; int max_retries; char *report_uri; bool remote_connections; + int handshake_wait_time; + int handshake_max_retries; } pmix_ptl_tcp_component_t; extern pmix_ptl_tcp_component_t mca_ptl_tcp_component; diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c index a880faa9c8..7f3138d52c 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c @@ -12,9 +12,10 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -60,6 +61,7 @@ #include "src/util/os_path.h" #include "src/util/parse_options.h" #include "src/util/pif.h" +#include "src/util/pmix_environ.h" #include "src/util/show_help.h" #include "src/util/strnlen.h" #include "src/common/pmix_iof.h" @@ -116,10 +118,13 @@ static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env); .session_filename = NULL, .nspace_filename = NULL, .system_filename = NULL, + .rendezvous_filename = NULL, .wait_to_connect = 4, .max_retries = 2, .report_uri = NULL, - .remote_connections = false + .remote_connections = false, + .handshake_wait_time = 4, + .handshake_max_retries = 2 }; static char **split_and_resolve(char **orig_str, char *name); @@ -148,7 +153,7 @@ static int component_register(void) (void)pmix_mca_base_component_var_register(component, "remote_connections", "Enable connections from remote tools", - PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, PMIX_INFO_LVL_2, PMIX_MCA_BASE_VAR_SCOPE_LOCAL, &mca_ptl_tcp_component.remote_connections); @@ -220,6 +225,20 @@ static int component_register(void) PMIX_MCA_BASE_VAR_SCOPE_READONLY, &mca_ptl_tcp_component.max_retries); + (void)pmix_mca_base_component_var_register(component, "handshake_wait_time", + "Number of seconds to wait for the server reply to the handshake request", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_4, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_ptl_tcp_component.handshake_wait_time); + + (void)pmix_mca_base_component_var_register(component, "handshake_max_retries", + "Number of times to retry the handshake request before giving up", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_4, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_ptl_tcp_component.handshake_max_retries); + return PMIX_SUCCESS; } @@ -233,31 +252,28 @@ static pmix_status_t component_open(void) /* check for environ-based directives * on system tmpdir to use */ - if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || + PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { mca_ptl_tcp_component.session_tmpdir = strdup(pmix_server_globals.tmpdir); } else { if (NULL != (tdir = getenv("PMIX_SERVER_TMPDIR"))) { mca_ptl_tcp_component.session_tmpdir = strdup(tdir); + } else { + mca_ptl_tcp_component.session_tmpdir = strdup(pmix_tmp_directory()); } } - if (NULL != (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) { - mca_ptl_tcp_component.system_tmpdir = strdup(tdir); - } - - if (NULL == (tdir = getenv("TMPDIR"))) { - if (NULL == (tdir = getenv("TEMP"))) { - if (NULL == (tdir = getenv("TMP"))) { - tdir = "/tmp"; - } + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || + PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + mca_ptl_tcp_component.system_tmpdir = strdup(pmix_server_globals.system_tmpdir); + } else { + if (NULL != (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) { + mca_ptl_tcp_component.system_tmpdir = strdup(tdir); + } else { + mca_ptl_tcp_component.system_tmpdir = strdup(pmix_tmp_directory()); } } - if (NULL == mca_ptl_tcp_component.session_tmpdir) { - mca_ptl_tcp_component.session_tmpdir = strdup(tdir); - } - if (NULL == mca_ptl_tcp_component.system_tmpdir) { - mca_ptl_tcp_component.system_tmpdir = strdup(tdir); - } + if (NULL != mca_ptl_tcp_component.report_uri && 0 != strcmp(mca_ptl_tcp_component.report_uri, "-") && 0 != strcmp(mca_ptl_tcp_component.report_uri, "+")) { @@ -271,12 +287,19 @@ pmix_status_t component_close(void) { if (NULL != mca_ptl_tcp_component.system_filename) { unlink(mca_ptl_tcp_component.system_filename); + free(mca_ptl_tcp_component.system_filename); } if (NULL != mca_ptl_tcp_component.session_filename) { unlink(mca_ptl_tcp_component.session_filename); + free(mca_ptl_tcp_component.session_filename); } if (NULL != mca_ptl_tcp_component.nspace_filename) { unlink(mca_ptl_tcp_component.nspace_filename); + free(mca_ptl_tcp_component.nspace_filename); + } + if (NULL != mca_ptl_tcp_component.rendezvous_filename) { + unlink(mca_ptl_tcp_component.rendezvous_filename); + free(mca_ptl_tcp_component.rendezvous_filename); } if (NULL != urifile) { /* remove the file */ @@ -301,19 +324,8 @@ static int component_query(pmix_mca_base_module_t **module, int *priority) static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env) { - char *evar; - - if (0 > asprintf(&evar, "PMIX_SERVER_TMPDIR=%s", mca_ptl_tcp_component.session_tmpdir)) { - return PMIX_ERR_NOMEM; - } - pmix_argv_append_nosize(env, evar); - free(evar); - - if (0 > asprintf(&evar, "PMIX_SYSTEM_TMPDIR=%s", mca_ptl_tcp_component.system_tmpdir)) { - return PMIX_ERR_NOMEM; - } - pmix_argv_append_nosize(env, evar); - free(evar); + pmix_setenv("PMIX_SERVER_TMPDIR", mca_ptl_tcp_component.session_tmpdir, true, env); + pmix_setenv("PMIX_SYSTEM_TMPDIR", mca_ptl_tcp_component.system_tmpdir, true, env); return PMIX_SUCCESS; } @@ -345,6 +357,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, char *prefix, myhost[PMIX_MAXHOSTNAMELEN]; char myconnhost[PMIX_MAXHOSTNAMELEN]; int myport; + pmix_kval_t *urikv; pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp setup_listener"); @@ -357,51 +370,54 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, /* scan the info keys and process any override instructions */ if (NULL != info) { for (n=0; n < ninfo; n++) { - if (0 == strcmp(info[n].key, PMIX_TCP_IF_INCLUDE)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IF_INCLUDE)) { if (NULL != mca_ptl_tcp_component.if_include) { free(mca_ptl_tcp_component.if_include); } mca_ptl_tcp_component.if_include = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_TCP_IF_EXCLUDE)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IF_EXCLUDE)) { if (NULL != mca_ptl_tcp_component.if_exclude) { free(mca_ptl_tcp_component.if_exclude); } mca_ptl_tcp_component.if_exclude = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_TCP_IPV4_PORT)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IPV4_PORT)) { mca_ptl_tcp_component.ipv4_port = info[n].value.data.integer; - } else if (0 == strcmp(info[n].key, PMIX_TCP_IPV6_PORT)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_IPV6_PORT)) { mca_ptl_tcp_component.ipv6_port = info[n].value.data.integer; - } else if (0 == strcmp(info[n].key, PMIX_TCP_DISABLE_IPV4)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_DISABLE_IPV4)) { mca_ptl_tcp_component.disable_ipv4_family = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_TCP_DISABLE_IPV6)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_DISABLE_IPV6)) { mca_ptl_tcp_component.disable_ipv6_family = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_REMOTE_CONNECTIONS)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_REMOTE_CONNECTIONS)) { mca_ptl_tcp_component.remote_connections = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_TCP_URI)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_URI)) { if (NULL != mca_ptl_tcp_component.super.uri) { free(mca_ptl_tcp_component.super.uri); } mca_ptl_tcp_component.super.uri = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_TCP_REPORT_URI)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_TCP_REPORT_URI)) { if (NULL != mca_ptl_tcp_component.report_uri) { free(mca_ptl_tcp_component.report_uri); } mca_ptl_tcp_component.report_uri = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_TMPDIR)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_TMPDIR)) { if (NULL != mca_ptl_tcp_component.session_tmpdir) { free(mca_ptl_tcp_component.session_tmpdir); } mca_ptl_tcp_component.session_tmpdir = strdup(info[n].value.data.string); - } else if (0 == strcmp(info[n].key, PMIX_SYSTEM_TMPDIR)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SYSTEM_TMPDIR)) { if (NULL != mca_ptl_tcp_component.system_tmpdir) { free(mca_ptl_tcp_component.system_tmpdir); } mca_ptl_tcp_component.system_tmpdir = strdup(info[n].value.data.string); } else if (0 == strcmp(info[n].key, PMIX_SERVER_TOOL_SUPPORT)) { session_tool = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_SYSTEM_SUPPORT)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_SYSTEM_SUPPORT)) { system_tool = PMIX_INFO_TRUE(&info[n]); - } + } else if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer) && + PMIX_CHECK_KEY(&info[n], PMIX_LAUNCHER_RENDEZVOUS_FILE)) { + mca_ptl_tcp_component.rendezvous_filename = strdup(info[n].value.data.string); + } } } @@ -625,6 +641,16 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "ptl:tcp URI %s", lt->uri); + /* save the URI internally so we can report it */ + urikv = PMIX_NEW(pmix_kval_t); + urikv->key = strdup(PMIX_SERVER_URI); + PMIX_VALUE_CREATE(urikv->value, 1); + PMIX_VALUE_LOAD(urikv->value, lt->uri, PMIX_STRING); + PMIX_GDS_STORE_KV(rc, pmix_globals.mypeer, + &pmix_globals.myid, PMIX_INTERNAL, + urikv); + PMIX_RELEASE(urikv); // maintain accounting + if (NULL != mca_ptl_tcp_component.report_uri) { /* if the string is a "-", then output to stdout */ if (0 == strcmp(mca_ptl_tcp_component.report_uri, "-")) { @@ -652,6 +678,38 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, } } + /* if we were given a rendezvous file, then drop it */ + if (NULL != mca_ptl_tcp_component.rendezvous_filename) { + FILE *fp; + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "WRITING RENDEZVOUS FILE %s", + mca_ptl_tcp_component.rendezvous_filename); + fp = fopen(mca_ptl_tcp_component.rendezvous_filename, "w"); + if (NULL == fp) { + pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.rendezvous_filename); + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.rendezvous_filename); + mca_ptl_tcp_component.rendezvous_filename = NULL; + goto sockerror; + } + + /* output my nspace and rank plus the URI */ + fprintf(fp, "%s\n", lt->uri); + /* add a flag that indicates we accept v3.0 protocols */ + fprintf(fp, "v%s\n", PMIX_VERSION); + fclose(fp); + /* set the file mode */ + if (0 != chmod(mca_ptl_tcp_component.rendezvous_filename, S_IRUSR | S_IWUSR | S_IRGRP)) { + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.rendezvous_filename); + mca_ptl_tcp_component.rendezvous_filename = NULL; + goto sockerror; + } + } + /* if we are going to support tools, then drop contact file(s) */ if (system_tool) { FILE *fp; @@ -906,14 +964,14 @@ static void connection_handler(int sd, short args, void *cbdata) char *nspace; uint32_t len, u32; size_t cnt, msglen, n; - uint8_t flag; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; bool found; pmix_rank_info_t *info; pmix_proc_t proc; pmix_info_t ginfo; pmix_proc_type_t proc_type; pmix_byte_object_t cred; + pmix_buffer_t buf; /* acquire the object */ PMIX_ACQUIRE_OBJECT(pnd); @@ -1008,7 +1066,7 @@ static void connection_handler(int sd, short args, void *cbdata) /* get the process type of the connecting peer */ if (1 <= cnt) { - memcpy(&flag, mg, 1); + memcpy(&pnd->flag, mg, 1); ++mg; --cnt; } else { @@ -1018,7 +1076,7 @@ static void connection_handler(int sd, short args, void *cbdata) goto error; } - if (0 == flag) { + if (0 == pnd->flag) { /* they must be a client, so get their nspace/rank */ proc_type = PMIX_PROC_CLIENT; PMIX_STRNLEN(msglen, mg, cnt); @@ -1045,7 +1103,7 @@ static void connection_handler(int sd, short args, void *cbdata) rc = PMIX_ERR_BAD_PARAM; goto error; } - } else if (1 == flag) { + } else if (1 == pnd->flag) { /* they are a tool */ proc_type = PMIX_PROC_TOOL; /* extract the uid/gid */ @@ -1071,7 +1129,7 @@ static void connection_handler(int sd, short args, void *cbdata) rc = PMIX_ERR_BAD_PARAM; goto error; } - } else if (2 == flag) { + } else if (2 == pnd->flag) { /* they are a launcher */ proc_type = PMIX_PROC_LAUNCHER; /* extract the uid/gid */ @@ -1097,8 +1155,95 @@ static void connection_handler(int sd, short args, void *cbdata) rc = PMIX_ERR_BAD_PARAM; goto error; } + } else if (3 == pnd->flag || 6 == pnd->flag) { + /* they are a tool or launcher that needs an identifier */ + if (3 == pnd->flag) { + proc_type = PMIX_PROC_TOOL; + } else { + proc_type = PMIX_PROC_LAUNCHER; + } + /* extract the uid/gid */ + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->uid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->gid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + /* they need an id */ + pnd->need_id = true; + } else if (4 == pnd->flag || 5 == pnd->flag || 7 == pnd->flag || 8 == pnd->flag) { + /* they are a tool or launcher that has an identifier - start with our ACLs */ + if (4 == pnd->flag || 5 == pnd->flag) { + proc_type = PMIX_PROC_TOOL; + } else { + proc_type = PMIX_PROC_LAUNCHER; + } + /* extract the uid/gid */ + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->uid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->gid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + PMIX_STRNLEN(msglen, mg, cnt); + if (msglen < cnt) { + nspace = mg; + mg += strlen(nspace) + 1; + cnt -= strlen(nspace) + 1; + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + + if (sizeof(pmix_rank_t) <= cnt) { + /* have to convert this to host order */ + memcpy(&u32, mg, sizeof(uint32_t)); + rank = ntohl(u32); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } } else { /* we don't know what they are! */ + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); rc = PMIX_ERR_NOT_SUPPORTED; free(msg); goto error; @@ -1123,7 +1268,7 @@ static void connection_handler(int sd, short args, void *cbdata) proc_type = proc_type | PMIX_PROC_V20; bfrops = "v20"; bftype = pmix_bfrops_globals.default_type; // we can't know any better - gds = NULL; + gds = "ds12,hash"; } else { int major; major = strtoul(version, NULL, 10); @@ -1133,6 +1278,7 @@ static void connection_handler(int sd, short args, void *cbdata) proc_type = proc_type | PMIX_PROC_V3; } else { free(msg); + PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); rc = PMIX_ERR_NOT_SUPPORTED; goto error; } @@ -1179,63 +1325,137 @@ static void connection_handler(int sd, short args, void *cbdata) } /* see if this is a tool connection request */ - if (0 != flag) { - /* does the server support tool connections? */ - if (NULL == pmix_host_server.tool_connected) { - /* send an error reply to the client */ - rc = PMIX_ERR_NOT_SUPPORTED; - goto error; + if (0 != pnd->flag) { + peer = PMIX_NEW(pmix_peer_t); + if (NULL == peer) { + /* probably cannot send an error reply if we are out of memory */ + free(msg); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; } - - if (PMIX_PROC_V3 & proc_type) { - /* the caller will have provided a flag indicating - * whether or not they have an assigned nspace/rank */ - if (cnt < 1) { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + pnd->peer = peer; + /* if this is a tool we launched, then the host may + * have already registered it as a client - so check + * to see if we already have a peer for it */ + if (5 == pnd->flag || 8 == pnd->flag) { + /* registration only adds the nspace and a rank in that + * nspace - it doesn't add the peer object to our array + * of local clients. So let's start by searching for + * the nspace object */ + nptr = NULL; + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { + if (0 == strcmp(tmp->nspace, nspace)) { + nptr = tmp; + break; + } + } + if (NULL == nptr) { + /* we don't know this namespace, reject it */ free(msg); /* send an error reply to the client */ - rc = PMIX_ERR_BAD_PARAM; + rc = PMIX_ERR_NOT_FOUND; goto error; } - memcpy(&flag, mg, 1); - ++mg; - --cnt; - if (flag) { - PMIX_STRNLEN(msglen, mg, cnt); - if (msglen < cnt) { - nspace = mg; - mg += strlen(nspace) + 1; - cnt -= strlen(nspace) + 1; - } else { - free(msg); - /* send an error reply to the client */ - rc = PMIX_ERR_BAD_PARAM; - goto error; - } - if (sizeof(pmix_rank_t) <= cnt) { - /* have to convert this to host order */ - memcpy(&u32, mg, sizeof(uint32_t)); - rank = ntohl(u32); - mg += sizeof(uint32_t); - cnt -= sizeof(uint32_t); - } else { - free(msg); - /* send an error reply to the client */ - rc = PMIX_ERR_BAD_PARAM; - goto error; + /* now look for the rank */ + info = NULL; + found = false; + PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) { + if (info->pname.rank == rank) { + found = true; + break; } + } + if (!found) { + /* rank unknown, reject it */ + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + PMIX_RETAIN(info); + peer->info = info; + PMIX_RETAIN(nptr); + } else { + nptr = PMIX_NEW(pmix_namespace_t); + if (NULL == nptr) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + PMIX_RELEASE(peer); + return; + } + } + peer->nptr = nptr; + /* select their bfrops compat module */ + peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(bfrops); + if (NULL == peer->nptr->compat.bfrops) { + PMIX_RELEASE(peer); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + /* set the buffer type */ + peer->nptr->compat.type = bftype; + n = 0; + /* if info structs need to be passed along, then unpack them */ + if (0 < cnt) { + int32_t foo; + PMIX_CONSTRUCT(&buf, pmix_buffer_t); + PMIX_LOAD_BUFFER(peer, &buf, mg, cnt); + foo = 1; + PMIX_BFROPS_UNPACK(rc, peer, &buf, &pnd->ninfo, &foo, PMIX_SIZE); + foo = (int32_t)pnd->ninfo; + /* if we have an identifier, then we leave room to pass it */ + if (!pnd->need_id) { + pnd->ninfo += 5; + } else { + pnd->ninfo += 3; + } + PMIX_INFO_CREATE(pnd->info, pnd->ninfo); + PMIX_BFROPS_UNPACK(rc, peer, &buf, pnd->info, &foo, PMIX_INFO); + n = foo; + } else { + if (!pnd->need_id) { pnd->ninfo = 5; } else { pnd->ninfo = 3; } - } else { - pnd->ninfo = 3; + PMIX_INFO_CREATE(pnd->info, pnd->ninfo); + } + + /* pass along the proc_type */ + pnd->proc_type = proc_type; + /* pass along the bfrop, buffer_type, and sec fields so + * we can assign them once we create a peer object */ + pnd->psec = strdup(sec); + if (NULL != gds) { + pnd->gds = strdup(gds); + } + + /* does the server support tool connections? */ + if (NULL == pmix_host_server.tool_connected) { + if (pnd->need_id) { + /* we need someone to provide the tool with an + * identifier and they aren't available */ + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_SUPPORTED; + PMIX_RELEASE(peer); + /* release the msg */ + free(msg); + goto error; + } else { + /* just process it locally */ + PMIX_LOAD_PROCID(&proc, nspace, rank); + cnct_cbfunc(PMIX_SUCCESS, &proc, (void*)pnd); + /* release the msg */ + free(msg); + return; + } } /* setup the info array to pass the relevant info * to the server */ - n = 0; - PMIX_INFO_CREATE(pnd->info, pnd->ninfo); /* provide the version */ PMIX_INFO_LOAD(&pnd->info[n], PMIX_VERSION_INFO, version, PMIX_STRING); ++n; @@ -1245,37 +1465,24 @@ static void connection_handler(int sd, short args, void *cbdata) /* and the group id */ PMIX_INFO_LOAD(&pnd->info[n], PMIX_GRPID, &pnd->gid, PMIX_UINT32); ++n; - /* if we have it, pass along our ID */ - if (flag) { + /* if we have it, pass along their ID */ + if (!pnd->need_id) { PMIX_INFO_LOAD(&pnd->info[n], PMIX_NSPACE, nspace, PMIX_STRING); ++n; PMIX_INFO_LOAD(&pnd->info[n], PMIX_RANK, &rank, PMIX_PROC_RANK); ++n; } - /* pass along the proc_type */ - pnd->proc_type = proc_type; - /* pass along the bfrop, buffer_type, and sec fields so - * we can assign them once we create a peer object */ - pnd->psec = strdup(sec); - if (NULL != bfrops) { - pnd->bfrops = strdup(bfrops); - } - pnd->buffer_type = bftype; - if (NULL != gds) { - pnd->gds = strdup(gds); - } /* release the msg */ free(msg); - /* request an nspace for this requestor - it will - * automatically be assigned rank=0 if the rank - * isn't already known */ + + /* pass it up for processing */ pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd); return; } /* see if we know this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, nspace)) { nptr = tmp; break; @@ -1443,12 +1650,16 @@ static void connection_handler(int sd, short args, void *cbdata) /* let the host server know that this client has connected */ if (NULL != pmix_host_server.client_connected) { - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.client_connected(&proc, peer->info->server_object, NULL, NULL); - if (PMIX_SUCCESS != rc) { + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { PMIX_ERROR_LOG(rc); + info->proc_cnt--; + pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); + PMIX_RELEASE(peer); + goto error; } } @@ -1483,7 +1694,7 @@ static void process_cbfunc(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cd->cbdata; - pmix_nspace_t *nptr; + pmix_namespace_t *nptr; pmix_rank_info_t *info; pmix_peer_t *peer; int rc; @@ -1500,6 +1711,7 @@ static void process_cbfunc(int sd, short args, void *cbdata) if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { PMIX_ERROR_LOG(rc); CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); PMIX_RELEASE(pnd); PMIX_RELEASE(cd); return; @@ -1507,24 +1719,41 @@ static void process_cbfunc(int sd, short args, void *cbdata) /* if the request failed, then we are done */ if (PMIX_SUCCESS != cd->status) { + PMIX_RELEASE(pnd->peer); PMIX_RELEASE(pnd); PMIX_RELEASE(cd); return; } - /* send the nspace back to the tool */ - if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) { - PMIX_ERROR_LOG(rc); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; + /* if we got an identifier, send it back to the tool */ + if (pnd->need_id) { + /* start with the nspace */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } + + /* now the rank, suitably converted */ + u32 = ntohl(cd->proc.rank); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } } /* send my nspace back to the tool */ if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1))) { PMIX_ERROR_LOG(rc); CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); PMIX_RELEASE(pnd); PMIX_RELEASE(cd); return; @@ -1535,61 +1764,41 @@ static void process_cbfunc(int sd, short args, void *cbdata) if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { PMIX_ERROR_LOG(rc); CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd->peer); PMIX_RELEASE(pnd); PMIX_RELEASE(cd); return; } - /* add this nspace to our pool */ - nptr = PMIX_NEW(pmix_nspace_t); - if (NULL == nptr) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - nptr->nspace = strdup(cd->proc.nspace); - pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); - /* add this tool rank to the nspace */ - info = PMIX_NEW(pmix_rank_info_t); - if (NULL == info) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - info->pname.nspace = strdup(cd->proc.nspace); - info->pname.rank = 0; - /* need to include the uid/gid for validation */ - info->uid = pnd->uid; - info->gid = pnd->gid; - pmix_list_append(&nptr->ranks, &info->super); + /* shortcuts */ + peer = (pmix_peer_t*)pnd->peer; + nptr = peer->nptr; - /* setup a peer object for this tool */ - peer = PMIX_NEW(pmix_peer_t); - if (NULL == peer) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; + /* if this tool wasn't initially registered as a client, + * then add some required structures */ + if (5 != pnd->flag && 8 != pnd->flag) { + PMIX_RETAIN(nptr); + nptr->nspace = strdup(cd->proc.nspace); + pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); + info = PMIX_NEW(pmix_rank_info_t); + info->pname.nspace = strdup(nptr->nspace); + info->pname.rank = cd->proc.rank; + info->uid = pnd->uid; + info->gid = pnd->gid; + pmix_list_append(&nptr->ranks, &info->super); + PMIX_RETAIN(info); + peer->info = info; } + /* mark the peer proc type */ peer->proc_type = pnd->proc_type; /* save the protocol */ peer->protocol = pnd->protocol; - /* add in the nspace pointer */ - PMIX_RETAIN(nptr); - peer->nptr = nptr; - PMIX_RETAIN(info); - peer->info = info; /* save the uid/gid */ - peer->epilog.uid = info->uid; - peer->epilog.gid = info->gid; - nptr->epilog.uid = info->uid; - nptr->epilog.gid = info->gid; + peer->epilog.uid = peer->info->uid; + peer->epilog.gid = peer->info->gid; + nptr->epilog.uid = peer->info->uid; + nptr->epilog.gid = peer->info->gid; peer->proc_cnt = 1; peer->sd = pnd->sd; @@ -1607,17 +1816,6 @@ static void process_cbfunc(int sd, short args, void *cbdata) * tool as we received this request via that channel, so simply * record it here for future use */ peer->nptr->compat.ptl = &pmix_ptl_tcp_module; - /* select their bfrops compat module */ - peer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(pnd->bfrops); - if (NULL == peer->nptr->compat.bfrops) { - PMIX_RELEASE(peer); - pmix_list_remove_item(&pmix_server_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); // will release the info object - CLOSE_THE_SOCKET(pnd->sd); - goto done; - } - /* set the buffer type */ - peer->nptr->compat.type = pnd->buffer_type; /* set the gds */ PMIX_INFO_LOAD(&ginfo, PMIX_GDS_MODULE, pnd->gds, PMIX_STRING); peer->nptr->compat.gds = pmix_gds_base_assign_module(&ginfo, 1); @@ -1718,7 +1916,8 @@ static void cnct_cbfunc(pmix_status_t status, return; } cd->status = status; - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + cd->proc.rank = proc->rank; cd->cbdata = cbdata; PMIX_THREADSHIFT(cd, process_cbfunc); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock.c index fcb15b0151..fc7b6da1c4 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock.c @@ -13,7 +13,7 @@ * Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -146,7 +146,7 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); } if (NULL == pmix_client_globals.myserver->nptr) { - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); } if (NULL == pmix_client_globals.myserver->nptr->nspace) { pmix_client_globals.myserver->nptr->nspace = strdup(uri[0]); diff --git a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock_component.c b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock_component.c index b09e147ace..7cb073db76 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock_component.c +++ b/opal/mca/pmix/pmix3x/pmix/src/mca/ptl/usock/ptl_usock_component.c @@ -15,6 +15,7 @@ * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -341,7 +342,7 @@ static void connection_handler(int sd, short args, void *cbdata) pmix_status_t rc; unsigned int rank; pmix_usock_hdr_t hdr; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; pmix_rank_info_t *info; pmix_peer_t *psave = NULL; bool found; @@ -481,6 +482,10 @@ static void connection_handler(int sd, short args, void *cbdata) cred.bytes = ptr; ptr += cred.size; len -= cred.size; + } else { + /* set cred pointer to NULL to guard against validation + * methods that assume a zero length credential is NULL */ + cred.bytes = NULL; } } @@ -541,7 +546,7 @@ static void connection_handler(int sd, short args, void *cbdata) /* see if we know this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, nspace)) { nptr = tmp; break; @@ -718,10 +723,10 @@ static void connection_handler(int sd, short args, void *cbdata) /* let the host server know that this client has connected */ if (NULL != pmix_host_server.client_connected) { - (void)strncpy(proc.nspace, psave->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, psave->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = psave->info->pname.rank; rc = pmix_host_server.client_connected(&proc, psave->info->server_object, NULL, NULL); - if (PMIX_SUCCESS != rc) { + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { PMIX_ERROR_LOG(rc); info->proc_cnt--; PMIX_RELEASE(info); diff --git a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c index bdfe4ebc41..87a1456f4d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c +++ b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_finalize.c @@ -52,6 +52,9 @@ extern bool pmix_init_called; void pmix_rte_finalize(void) { + int i; + pmix_notify_caddy_t *cd; + if( --pmix_initialized != 0 ) { if( pmix_initialized < 0 ) { fprintf(stderr, "PMIx Finalize called too many times\n"); @@ -104,9 +107,10 @@ void pmix_rte_finalize(void) PMIX_RELEASE(pmix_globals.mypeer); PMIX_DESTRUCT(&pmix_globals.events); PMIX_LIST_DESTRUCT(&pmix_globals.cached_events); - { - pmix_notify_caddy_t *cd; - while (NULL != (cd=(pmix_notify_caddy_t *)pmix_ring_buffer_pop(&pmix_globals.notifications))) { + /* clear any notifications */ + for (i=0; i < pmix_globals.max_events; i++) { + pmix_hotel_checkout_and_return_occupant(&pmix_globals.notifications, i, (void**)&cd); + if (NULL != cd) { PMIX_RELEASE(cd); } } diff --git a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c index 7a9fd4d872..d1803de704 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c +++ b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_init.c @@ -15,7 +15,7 @@ * Copyright (c) 2009 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2010-2015 Los Alamos National Security, LLC. * All rights reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -70,6 +70,8 @@ PMIX_EXPORT bool pmix_init_called = false; PMIX_EXPORT pmix_globals_t pmix_globals = { .init_cntr = 0, .mypeer = NULL, + .hostname = NULL, + .nodeid = UINT32_MAX, .pindex = 0, .evbase = NULL, .external_evbase = false, @@ -80,6 +82,15 @@ PMIX_EXPORT pmix_globals_t pmix_globals = { }; +static void _notification_eviction_cbfunc(struct pmix_hotel_t *hotel, + int room_num, + void *occupant) +{ + pmix_notify_caddy_t *cache = (pmix_notify_caddy_t*)occupant; + PMIX_RELEASE(cache); +} + + int pmix_rte_init(pmix_proc_type_t type, pmix_info_t info[], size_t ninfo, pmix_ptl_cbfunc_t cbfunc) @@ -87,6 +98,7 @@ int pmix_rte_init(pmix_proc_type_t type, int ret, debug_level; char *error = NULL, *evar; size_t n; + char hostname[PMIX_MAXHOSTNAMELEN]; if( ++pmix_initialized != 1 ) { if( pmix_initialized < 1 ) { @@ -147,6 +159,8 @@ int pmix_rte_init(pmix_proc_type_t type, } /* setup the globals structure */ + gethostname(hostname, PMIX_MAXHOSTNAMELEN); + pmix_globals.hostname = strdup(hostname); memset(&pmix_globals.myid.nspace, 0, PMIX_MAX_NSLEN+1); pmix_globals.myid.rank = PMIX_RANK_INVALID; PMIX_CONSTRUCT(&pmix_globals.events, pmix_events_t); @@ -154,8 +168,15 @@ int pmix_rte_init(pmix_proc_type_t type, pmix_globals.event_window.tv_usec = 0; PMIX_CONSTRUCT(&pmix_globals.cached_events, pmix_list_t); /* construct the global notification ring buffer */ - PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_ring_buffer_t); - pmix_ring_buffer_init(&pmix_globals.notifications, 256); + PMIX_CONSTRUCT(&pmix_globals.notifications, pmix_hotel_t); + ret = pmix_hotel_init(&pmix_globals.notifications, pmix_globals.max_events, + pmix_globals.evbase, pmix_globals.event_eviction_time, + _notification_eviction_cbfunc); + if (PMIX_SUCCESS != ret) { + error = "notification hotel init"; + goto return_error; + } + /* and setup the iof request tracking list */ PMIX_CONSTRUCT(&pmix_globals.iof_requests, pmix_list_t); @@ -223,7 +244,7 @@ int pmix_rte_init(pmix_proc_type_t type, pmix_globals.mypeer->proc_type = type | PMIX_PROC_V3; /* create an nspace object for ourselves - we will * fill in the nspace name later */ - pmix_globals.mypeer->nptr = PMIX_NEW(pmix_nspace_t); + pmix_globals.mypeer->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == pmix_globals.mypeer->nptr) { PMIX_RELEASE(pmix_globals.mypeer); ret = PMIX_ERR_NOMEM; @@ -233,9 +254,19 @@ int pmix_rte_init(pmix_proc_type_t type, /* scan incoming info for directives */ if (NULL != info) { for (n=0; n < ninfo; n++) { - if (0 == strcmp(PMIX_EVENT_BASE, info[n].key)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_BASE)) { pmix_globals.evbase = (pmix_event_base_t*)info[n].value.data.ptr; pmix_globals.external_evbase = true; + } else if (PMIX_CHECK_KEY(&info[n], PMIX_HOSTNAME)) { + if (NULL != pmix_globals.hostname) { + free(pmix_globals.hostname); + } + pmix_globals.hostname = strdup(info[n].value.data.string); + } else if (PMIX_CHECK_KEY(&info[n], PMIX_NODEID)) { + PMIX_VALUE_GET_NUMBER(ret, &info[n].value, pmix_globals.nodeid, uint32_t); + if (PMIX_SUCCESS != ret) { + goto return_error; + } } } } diff --git a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_params.c b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_params.c index 4524c216a9..8d49e8bdaa 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_params.c +++ b/opal/mca/pmix/pmix3x/pmix/src/runtime/pmix_params.c @@ -19,9 +19,9 @@ * Copyright (c) 2014 Hochschule Esslingen. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -242,6 +242,30 @@ pmix_status_t pmix_register_params(void) PMIX_INFO_LVL_9, PMIX_MCA_BASE_VAR_SCOPE_READONLY, &pmix_globals.timestamp_output); + /* max size of the notification hotel */ + pmix_globals.max_events = 512; + (void) pmix_mca_base_var_register ("pmix", "pmix", "max", "events", + "Maximum number of event notifications to cache", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_globals.max_events); + + /* how long to cache an event */ + pmix_globals.event_eviction_time = 120; + (void) pmix_mca_base_var_register ("pmix", "pmix", "event", "eviction_time", + "Maximum number of seconds to cache an event", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_globals.event_eviction_time); + + /* max number of IOF messages to cache */ + pmix_server_globals.max_iof_cache = 1024 * 1024; + (void) pmix_mca_base_var_register ("pmix", "pmix", "max", "iof_cache", + "Maximum number of IOF messages to cache", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_1, PMIX_MCA_BASE_VAR_SCOPE_ALL, + &pmix_server_globals.max_iof_cache); + return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c index 075c877bc1..38b85c0175 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c +++ b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c @@ -1,13 +1,14 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. * All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. + * Copyright (c) 2016-2018 IBM Corporation. All rights reserved. + * Copyright (c) 2018 Cisco Systems, Inc. All rights reserved * $COPYRIGHT$ * * Additional copyrights may follow @@ -86,18 +87,8 @@ static char *gds_mode = NULL; static pid_t mypid; // local functions for connection support -static void iof_eviction_cbfunc(struct pmix_hotel_t *hotel, - int room_num, - void *occupant) -{ - pmix_setup_caddy_t *cache = (pmix_setup_caddy_t*)occupant; - PMIX_RELEASE(cache); -} - pmix_status_t pmix_server_initialize(void) { - pmix_status_t rc; - /* setup the server-specific globals */ PMIX_CONSTRUCT(&pmix_server_globals.clients, pmix_pointer_array_t); pmix_pointer_array_init(&pmix_server_globals.clients, 1, INT_MAX, 1); @@ -107,15 +98,7 @@ pmix_status_t pmix_server_initialize(void) PMIX_CONSTRUCT(&pmix_server_globals.events, pmix_list_t); PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t); PMIX_CONSTRUCT(&pmix_server_globals.nspaces, pmix_list_t); - PMIX_CONSTRUCT(&pmix_server_globals.iof, pmix_hotel_t); - rc = pmix_hotel_init(&pmix_server_globals.iof, PMIX_IOF_HOTEL_SIZE, - pmix_globals.evbase, PMIX_IOF_MAX_STAY, - iof_eviction_cbfunc); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } + PMIX_CONSTRUCT(&pmix_server_globals.iof, pmix_list_t); pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server init called"); @@ -182,7 +165,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, size_t n, m; pmix_kval_t *kv; bool protect, nspace_given = false, rank_given = false; - bool topology_req = false; pmix_info_t ginfo; char *protected[] = { PMIX_USERID, @@ -213,6 +195,8 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, } } else if (0 == strncmp(info[n].key, PMIX_SERVER_TMPDIR, PMIX_MAX_KEYLEN)) { pmix_server_globals.tmpdir = strdup(info[n].value.data.string); + } else if (0 == strncmp(info[n].key, PMIX_SYSTEM_TMPDIR, PMIX_MAX_KEYLEN)) { + pmix_server_globals.system_tmpdir = strdup(info[n].value.data.string); } } } @@ -223,6 +207,13 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, pmix_server_globals.tmpdir = strdup(evar); } } + if (NULL == pmix_server_globals.system_tmpdir) { + if (NULL == (evar = getenv("PMIX_SYSTEM_TMPDIR"))) { + pmix_server_globals.system_tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.system_tmpdir = strdup(evar); + } + } /* setup the runtime - this init's the globals, * opens and initializes the required frameworks */ @@ -299,17 +290,11 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, if (NULL != info) { for (n=0; n < ninfo; n++) { if (0 == strncmp(info[n].key, PMIX_SERVER_NSPACE, PMIX_MAX_KEYLEN)) { - (void)strncpy(pmix_globals.myid.nspace, info[n].value.data.string, PMIX_MAX_NSLEN); + pmix_strncpy(pmix_globals.myid.nspace, info[n].value.data.string, PMIX_MAX_NSLEN); nspace_given = true; } else if (0 == strncmp(info[n].key, PMIX_SERVER_RANK, PMIX_MAX_KEYLEN)) { pmix_globals.myid.rank = info[n].value.data.rank; rank_given = true; - } else if (0 == strncmp(info[n].key, PMIX_TOPOLOGY, PMIX_MAX_KEYLEN) || - 0 == strncmp(info[n].key, PMIX_TOPOLOGY_XML, PMIX_MAX_KEYLEN) || - 0 == strncmp(info[n].key, PMIX_TOPOLOGY_FILE, PMIX_MAX_KEYLEN) || - 0 == strncmp(info[n].key, PMIX_HWLOC_XML_V1, PMIX_MAX_KEYLEN) || - 0 == strncmp(info[n].key, PMIX_HWLOC_XML_V2, PMIX_MAX_KEYLEN)) { - topology_req = true; } else { /* check the list of protected keys */ protect = false; @@ -343,9 +328,9 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, /* look for our namespace, if one was given */ if (NULL == (evar = getenv("PMIX_SERVER_NAMESPACE"))) { /* use a fake namespace */ - (void)strncpy(pmix_globals.myid.nspace, "pmix-server", PMIX_MAX_NSLEN); + pmix_strncpy(pmix_globals.myid.nspace, "pmix-server", PMIX_MAX_NSLEN); } else { - (void)strncpy(pmix_globals.myid.nspace, evar, PMIX_MAX_NSLEN); + pmix_strncpy(pmix_globals.myid.nspace, evar, PMIX_MAX_NSLEN); } } if (!rank_given) { @@ -367,7 +352,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, rinfo = pmix_globals.mypeer->info; } if (NULL == pmix_globals.mypeer->nptr) { - pmix_globals.mypeer->nptr = PMIX_NEW(pmix_nspace_t); + pmix_globals.mypeer->nptr = PMIX_NEW(pmix_namespace_t); /* ensure our own nspace is first on the list */ PMIX_RETAIN(pmix_globals.mypeer->nptr); pmix_list_prepend(&pmix_server_globals.nspaces, &pmix_globals.mypeer->nptr->super); @@ -391,11 +376,9 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, } /* if requested, setup the topology */ - if (topology_req) { - if (PMIX_SUCCESS != (rc = pmix_hwloc_get_topology(info, ninfo))) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } + if (PMIX_SUCCESS != (rc = pmix_hwloc_get_topology(info, ninfo))) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; } /* open the psensor framework */ @@ -442,8 +425,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) { int i; pmix_peer_t *peer; - pmix_nspace_t *ns; - pmix_setup_caddy_t *cd; + pmix_namespace_t *ns; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { @@ -471,14 +453,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) pmix_ptl_base_stop_listening(); - /* cleanout any IOF */ - for (i=0; i < PMIX_IOF_HOTEL_SIZE; i++) { - pmix_hotel_checkout_and_return_occupant(&pmix_server_globals.iof, i, (void**)&cd); - if (NULL != cd) { - PMIX_RELEASE(cd); - } - } - PMIX_DESTRUCT(&pmix_server_globals.iof); for (i=0; i < pmix_server_globals.clients.size; i++) { if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) { /* ensure that we do the specified cleanup - if this is an @@ -494,13 +468,14 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs); PMIX_LIST_DESTRUCT(&pmix_server_globals.gdata); PMIX_LIST_DESTRUCT(&pmix_server_globals.events); - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { /* ensure that we do the specified cleanup - if this is an * abnormal termination, then the nspace object may not be * at zero refcount */ pmix_execute_epilog(&ns->epilog); } PMIX_LIST_DESTRUCT(&pmix_server_globals.nspaces); + PMIX_LIST_DESTRUCT(&pmix_server_globals.iof); pmix_hwloc_cleanup(); @@ -548,7 +523,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) static void _register_nspace(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; pmix_status_t rc; size_t i; @@ -559,14 +534,14 @@ static void _register_nspace(int sd, short args, void *cbdata) /* see if we already have this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, cd->proc.nspace)) { nptr = tmp; break; } } if (NULL == nptr) { - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); if (NULL == nptr) { rc = PMIX_ERR_NOMEM; goto release; @@ -611,7 +586,7 @@ static void _register_nspace(int sd, short args, void *cbdata) } /* setup the data for a job */ -PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int nlocalprocs, +PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const pmix_nspace_t nspace, int nlocalprocs, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { @@ -625,7 +600,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int n PMIX_RELEASE_THREAD(&pmix_global_lock); cd = PMIX_NEW(pmix_setup_caddy_t); - (void)strncpy(cd->proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, nspace, PMIX_MAX_NSLEN); cd->nlocalprocs = nlocalprocs; cd->opcbfunc = cbfunc; cd->cbdata = cbdata; @@ -641,10 +616,106 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const char nspace[], int n return PMIX_SUCCESS; } +void pmix_server_purge_events(pmix_peer_t *peer, + pmix_proc_t *proc) +{ + pmix_regevents_info_t *reginfo, *regnext; + pmix_peer_events_info_t *prev, *pnext; + pmix_iof_req_t *req, *nxt; + int i; + pmix_notify_caddy_t *ncd; + size_t n, m, p, ntgs; + pmix_proc_t *tgs, *tgt; + pmix_dmdx_local_t *dlcd, *dnxt; + + /* since the client is finalizing, remove them from any event + * registrations they may still have on our list */ + PMIX_LIST_FOREACH_SAFE(reginfo, regnext, &pmix_server_globals.events, pmix_regevents_info_t) { + PMIX_LIST_FOREACH_SAFE(prev, pnext, ®info->peers, pmix_peer_events_info_t) { + if ((NULL != peer && prev->peer == peer) || + (NULL != proc && PMIX_CHECK_PROCID(proc, &prev->peer->info->pname))) { + pmix_list_remove_item(®info->peers, &prev->super); + PMIX_RELEASE(prev); + if (0 == pmix_list_get_size(®info->peers)) { + pmix_list_remove_item(&pmix_server_globals.events, ®info->super); + PMIX_RELEASE(reginfo); + break; + } + } + } + } + + /* since the client is finalizing, remove them from any IOF + * registrations they may still have on our list */ + PMIX_LIST_FOREACH_SAFE(req, nxt, &pmix_globals.iof_requests, pmix_iof_req_t) { + if ((NULL != peer && PMIX_CHECK_PROCID(&req->peer->info->pname, &peer->info->pname)) || + (NULL != proc && PMIX_CHECK_PROCID(&req->peer->info->pname, proc))) { + pmix_list_remove_item(&pmix_globals.iof_requests, &req->super); + PMIX_RELEASE(req); + } + } + + /* see if this proc is involved in any direct modex requests */ + PMIX_LIST_FOREACH_SAFE(dlcd, dnxt, &pmix_server_globals.local_reqs, pmix_dmdx_local_t) { + if ((NULL != peer && PMIX_CHECK_PROCID(&peer->info->pname, &dlcd->proc)) || + (NULL != proc && PMIX_CHECK_PROCID(proc, &dlcd->proc))) { + /* cleanup this request */ + pmix_list_remove_item(&pmix_server_globals.local_reqs, &dlcd->super); + /* we can release the dlcd item here because we are not + * releasing the tracker held by the host - we are only + * releasing one item on that tracker */ + PMIX_RELEASE(dlcd); + } + } + + /* purge this client from any cached notifications */ + for (i=0; i < pmix_globals.max_events; i++) { + pmix_hotel_knock(&pmix_globals.notifications, i, (void**)&ncd); + if (NULL != ncd && NULL != ncd->targets && 0 < ncd->ntargets) { + tgt = NULL; + for (n=0; n < ncd->ntargets; n++) { + if ((NULL != peer && PMIX_CHECK_PROCID(&peer->info->pname, &ncd->targets[n])) || + (NULL != proc && PMIX_CHECK_PROCID(proc, &ncd->targets[n]))) { + tgt = &ncd->targets[n]; + break; + } + } + if (NULL != tgt) { + /* if this client was the only target, then just + * evict the notification */ + if (1 == ncd->ntargets) { + pmix_hotel_checkout(&pmix_globals.notifications, i); + PMIX_RELEASE(ncd); + } else if (PMIX_RANK_WILDCARD == tgt->rank && + NULL != proc && PMIX_RANK_WILDCARD == proc->rank) { + /* we have to remove this target, but leave the rest */ + ntgs = ncd->ntargets - 1; + PMIX_PROC_CREATE(tgs, ntgs); + p=0; + for (m=0; m < ncd->ntargets; m++) { + if (tgt != &ncd->targets[m]) { + memcpy(&tgs[p], &ncd->targets[n], sizeof(pmix_proc_t)); + ++p; + } + } + PMIX_PROC_FREE(ncd->targets, ncd->ntargets); + ncd->targets = tgs; + ncd->ntargets = ntgs; + } + } + } + } + + if (NULL != peer) { + /* ensure we honor any peer-level epilog requests */ + pmix_execute_epilog(&peer->epilog); + } +} + static void _deregister_nspace(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; - pmix_nspace_t *tmp; + pmix_namespace_t *tmp; pmix_status_t rc; PMIX_ACQUIRE_OBJECT(cd); @@ -653,15 +724,22 @@ static void _deregister_nspace(int sd, short args, void *cbdata) "pmix:server _deregister_nspace %s", cd->proc.nspace); - /* release any job-level messaging resources */ + /* release any job-level network resources */ pmix_pnet.deregister_nspace(cd->proc.nspace); /* let our local storage clean up */ PMIX_GDS_DEL_NSPACE(rc, cd->proc.nspace); + /* remove any event registrations, IOF registrations, and + * cached notifications targeting procs from this nspace */ + pmix_server_purge_events(NULL, &cd->proc); + /* release this nspace */ - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { - if (0 == strcmp(tmp->nspace, cd->proc.nspace)) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { + if (PMIX_CHECK_NSPACE(tmp->nspace, cd->proc.nspace)) { + /* perform any nspace-level epilog */ + pmix_execute_epilog(&tmp->epilog); + /* remove and release it */ pmix_list_remove_item(&pmix_server_globals.nspaces, &tmp->super); PMIX_RELEASE(tmp); break; @@ -675,7 +753,7 @@ static void _deregister_nspace(int sd, short args, void *cbdata) PMIX_RELEASE(cd); } -PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[], +PMIX_EXPORT void PMIx_server_deregister_nspace(const pmix_nspace_t nspace, pmix_op_cbfunc_t cbfunc, void *cbdata) { @@ -695,8 +773,8 @@ PMIX_EXPORT void PMIx_server_deregister_nspace(const char nspace[], } PMIX_RELEASE_THREAD(&pmix_global_lock); - cd = PMIX_NEW(pmix_setup_caddy_t); - (void)strncpy(cd->proc.nspace, nspace, PMIX_MAX_NSLEN); + cd = PMIX_NEW(pmix_setup_caddy_t); + PMIX_LOAD_PROCID(&cd->proc, nspace, PMIX_RANK_WILDCARD); cd->opcbfunc = cbfunc; cd->cbdata = cbdata; @@ -786,7 +864,7 @@ void pmix_server_execute_collective(int sd, short args, void *cbdata) } if (trk->hybrid || first) { /* setup the nspace */ - (void)strncpy(proc.nspace, cd->peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, cd->peer->info->pname.nspace, PMIX_MAX_NSLEN); first = false; } proc.rank = cd->peer->info->pname.rank; @@ -863,7 +941,7 @@ static void _register_client(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_rank_info_t *info, *iptr; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; pmix_server_trkr_t *trk; pmix_trkr_caddy_t *tcd; bool all_def; @@ -873,19 +951,20 @@ static void _register_client(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(cd); pmix_output_verbose(2, pmix_server_globals.base_output, - "pmix:server _register_client for nspace %s rank %d", - cd->proc.nspace, cd->proc.rank); + "pmix:server _register_client for nspace %s rank %d %s object", + cd->proc.nspace, cd->proc.rank, + (NULL == cd->server_object) ? "NULL" : "NON-NULL"); /* see if we already have this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(ns->nspace, cd->proc.nspace)) { nptr = ns; break; } } if (NULL == nptr) { - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); if (NULL == nptr) { rc = PMIX_ERR_NOMEM; goto cleanup; @@ -932,7 +1011,7 @@ static void _register_client(int sd, short args, void *cbdata) * if the nspaces are all defined */ if (all_def) { /* so far, they have all been defined - check this one */ - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 < ns->nlocalprocs && 0 == strcmp(trk->pcs[i].nspace, ns->nspace)) { all_def = ns->all_registered; @@ -1002,7 +1081,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, if (NULL == cd) { return PMIX_ERR_NOMEM; } - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); cd->proc.rank = proc->rank; cd->uid = uid; cd->gid = gid; @@ -1020,7 +1099,7 @@ static void _deregister_client(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_rank_info_t *info; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; pmix_peer_t *peer; PMIX_ACQUIRE_OBJECT(cd); @@ -1031,7 +1110,7 @@ static void _deregister_client(int sd, short args, void *cbdata) /* see if we already have this nspace */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, cd->proc.nspace)) { nptr = tmp; break; @@ -1073,6 +1152,12 @@ static void _deregister_client(int sd, short args, void *cbdata) pmix_pnet.child_finalized(&cd->proc); pmix_psensor.stop(peer, NULL); } + /* honor any registered epilogs */ + pmix_execute_epilog(&peer->epilog); + /* ensure we close the socket to this peer so we don't + * generate "connection lost" events should it be + * subsequently "killed" by the host */ + CLOSE_THE_SOCKET(peer->sd); } if (nptr->nlocalprocs == nptr->nfinalized) { pmix_pnet.local_app_finalized(nptr); @@ -1116,7 +1201,7 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc, } return; } - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); cd->proc.rank = proc->rank; cd->opcbfunc = cbfunc; cd->cbdata = cbdata; @@ -1204,7 +1289,7 @@ static void _dmodex_req(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_rank_info_t *info, *iptr; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; char *data = NULL; size_t sz = 0; pmix_dmdx_remote_t *dcd; @@ -1224,7 +1309,7 @@ static void _dmodex_req(int sd, short args, void *cbdata) * been informed of it - so first check to see if we know * about this nspace yet */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(ns->nspace, cd->proc.nspace)) { nptr = ns; break; @@ -1354,7 +1439,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_dmodex_request(const pmix_proc_t *proc, proc->nspace, proc->rank); cd = PMIX_NEW(pmix_setup_caddy_t); - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); cd->proc.rank = proc->rank; cd->cbfunc = cbfunc; cd->cbdata = cbdata; @@ -1372,7 +1457,7 @@ static void _store_internal(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(cd); - (void)strncpy(proc.nspace, cd->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, cd->pname.nspace, PMIX_MAX_NSLEN); proc.rank = cd->pname.rank; PMIX_GDS_STORE_KV(cd->status, pmix_globals.mypeer, &proc, PMIX_INTERNAL, cd->kv); @@ -1382,7 +1467,7 @@ static void _store_internal(int sd, short args, void *cbdata) } PMIX_EXPORT pmix_status_t PMIx_Store_internal(const pmix_proc_t *proc, - const char *key, pmix_value_t *val) + const pmix_key_t key, pmix_value_t *val) { pmix_shift_caddy_t *cd; pmix_status_t rc; @@ -1496,7 +1581,7 @@ static void _setup_app(int sd, short args, void *cbdata) } n = 0; PMIX_LIST_FOREACH(kv, &ilist, pmix_kval_t) { - (void)strncpy(fcd->info[n].key, kv->key, PMIX_MAX_KEYLEN); + pmix_strncpy(fcd->info[n].key, kv->key, PMIX_MAX_KEYLEN); pmix_value_xfer(&fcd->info[n].value, kv->value); ++n; } @@ -1520,7 +1605,7 @@ static void _setup_app(int sd, short args, void *cbdata) PMIX_RELEASE(cd); } -pmix_status_t PMIx_server_setup_application(const char nspace[], +pmix_status_t PMIx_server_setup_application(const pmix_nspace_t nspace, pmix_info_t info[], size_t ninfo, pmix_setup_application_cbfunc_t cbfunc, void *cbdata) { @@ -1571,7 +1656,7 @@ static void _setup_local_support(int sd, short args, void *cbdata) PMIX_RELEASE(cd); } -pmix_status_t PMIx_server_setup_local_support(const char nspace[], +pmix_status_t PMIx_server_setup_local_support(const pmix_nspace_t nspace, pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { @@ -1609,7 +1694,7 @@ static void _iofdeliver(int sd, short args, void *cbdata) pmix_buffer_t *msg; bool found = false; bool cached = false; - int ignore; + pmix_iof_cache_t *iof; pmix_output_verbose(2, pmix_server_globals.iof_output, "PMIX:SERVER delivering IOF from %s on channel %0x", @@ -1623,17 +1708,16 @@ static void _iofdeliver(int sd, short args, void *cbdata) continue; } /* see if the source matches the request */ - if (0 != strncmp(cd->procs->nspace, req->pname.nspace, PMIX_MAX_NSLEN) || - (PMIX_RANK_WILDCARD != req->pname.rank && cd->procs->rank != req->pname.rank)) { + if (!PMIX_CHECK_PROCID(cd->procs, &req->pname)) { continue; } /* never forward back to the source! This can happen if the source - * is a launcher */ + * is a launcher - also, never forward to a peer that is no + * longer with us */ if (NULL == req->peer->info || req->peer->finalized) { continue; } - if (0 == strncmp(cd->procs->nspace, req->peer->info->pname.nspace, PMIX_MAX_NSLEN) && - cd->procs->rank == req->peer->info->pname.rank) { + if (PMIX_CHECK_PROCID(cd->procs, &req->peer->info->pname)) { continue; } found = true; @@ -1674,15 +1758,21 @@ static void _iofdeliver(int sd, short args, void *cbdata) /* if nobody has registered for this yet, then cache it */ if (!found) { - /* add this output to our hotel so it is cached until someone - * registers to receive it */ - if (PMIX_SUCCESS != (rc = pmix_hotel_checkin(&pmix_server_globals.iof, cd, &ignore))) { - /* we can't cache it for some reason */ - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(cd); - return; + pmix_output_verbose(2, pmix_server_globals.iof_output, + "PMIx:SERVER caching IOF"); + if (pmix_server_globals.max_iof_cache == pmix_list_get_size(&pmix_server_globals.iof)) { + /* remove the oldest cached message */ + iof = (pmix_iof_cache_t*)pmix_list_remove_first(&pmix_server_globals.iof); + PMIX_RELEASE(iof); } - cached = true; + /* add this output to our cache so it is cached until someone + * registers to receive it */ + iof = PMIX_NEW(pmix_iof_cache_t); + memcpy(&iof->source, cd->procs, sizeof(pmix_proc_t)); + iof->channel = cd->channels; + iof->bo = cd->bo; + cd->bo = NULL; // protect the data + pmix_list_append(&pmix_server_globals.iof, &iof->super); } @@ -1690,11 +1780,6 @@ static void _iofdeliver(int sd, short args, void *cbdata) cd->opcbfunc(rc, cd->cbdata); } if (!cached) { - if (NULL != cd->info) { - PMIX_INFO_FREE(cd->info, cd->ninfo); - } - PMIX_PROC_FREE(cd->procs, 1); - PMIX_BYTE_OBJECT_FREE(cd->bo, 1); PMIX_RELEASE(cd); } } @@ -1720,7 +1805,8 @@ pmix_status_t PMIx_server_IOF_deliver(const pmix_proc_t *source, PMIX_RELEASE(cd); return PMIX_ERR_NOMEM; } - (void)strncpy(cd->procs[0].nspace, source->nspace, PMIX_MAX_NSLEN); + cd->nprocs = 1; + pmix_strncpy(cd->procs[0].nspace, source->nspace, PMIX_MAX_NSLEN); cd->procs[0].rank = source->rank; cd->channels = channel; PMIX_BYTE_OBJECT_CREATE(cd->bo, 1); @@ -1728,9 +1814,9 @@ pmix_status_t PMIx_server_IOF_deliver(const pmix_proc_t *source, PMIX_RELEASE(cd); return PMIX_ERR_NOMEM; } + cd->nbo = 1; cd->bo[0].bytes = (char*)malloc(bo->size); if (NULL == cd->bo[0].bytes) { - PMIX_BYTE_OBJECT_FREE(cd->bo, 1); PMIX_RELEASE(cd); return PMIX_ERR_NOMEM; } @@ -1739,7 +1825,6 @@ pmix_status_t PMIx_server_IOF_deliver(const pmix_proc_t *source, if (0 < ninfo) { PMIX_INFO_CREATE(cd->info, ninfo); if (NULL == cd->info) { - PMIX_BYTE_OBJECT_FREE(cd->bo, 1); PMIX_RELEASE(cd); return PMIX_ERR_NOMEM; } @@ -1803,7 +1888,7 @@ static void clct_complete(pmix_status_t status, /* transfer the results */ n=0; PMIX_LIST_FOREACH(kv, &cd->payload, pmix_kval_t) { - (void)strncpy(cd->info[n].key, kv->key, PMIX_MAX_KEYLEN); + pmix_strncpy(cd->info[n].key, kv->key, PMIX_MAX_KEYLEN); rc = pmix_value_xfer(&cd->info[n].value, kv->value); if (PMIX_SUCCESS != rc) { PMIX_INFO_FREE(cd->info, cd->ninfo); @@ -1835,10 +1920,10 @@ static void clct_complete(pmix_status_t status, static void clct(int sd, short args, void *cbdata) { pmix_inventory_rollup_t *cd = (pmix_inventory_rollup_t*)cbdata; - pmix_status_t rc; #if PMIX_HAVE_HWLOC /* if we don't know our topology, we better get it now */ + pmix_status_t rc; if (NULL == pmix_hwloc_topology) { if (PMIX_SUCCESS != (rc = pmix_hwloc_get_topology(NULL, 0))) { PMIX_ERROR_LOG(rc); @@ -2003,6 +2088,10 @@ static void connection_cleanup(int sd, short args, void *cbdata) { pmix_server_caddy_t *cd = (pmix_server_caddy_t*)cbdata; + /* ensure that we know the peer has finalized else we + * will generate an event - yes, it should have been + * done, but it is REALLY important that it be set */ + cd->peer->finalized = true; pmix_ptl_base_lost_connection(cd->peer, PMIX_SUCCESS); /* cleanup the caddy */ PMIX_RELEASE(cd); @@ -2040,10 +2129,6 @@ static void op_cbfunc2(pmix_status_t status, void *cbdata) PMIX_RELEASE(reply); } - /* ensure that we know the peer has finalized else we - * will generate an event - yes, it should have been - * done, but it is REALLY important that it be set */ - cd->peer->finalized = true; /* cleanup any lingering references to this peer - note * that we cannot call the lost_connection function * directly as we need the connection to still @@ -2073,40 +2158,47 @@ static void _spcb(int sd, short args, void *cbdata) PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, &cd->status, 1, PMIX_STATUS); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); goto cleanup; } - if (PMIX_SUCCESS == cd->status) { - /* pass back the name of the nspace */ - PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, &cd->pname.nspace, 1, PMIX_STRING); - /* add the job-level info, if we have it */ - (void)strncpy(proc.nspace, cd->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = PMIX_RANK_WILDCARD; - /* this is going to a local client, so let the gds - * have the option of returning a copy of the data, - * or a pointer to local storage */ - PMIX_CONSTRUCT(&cb, pmix_cb_t); - cb.proc = &proc; - cb.scope = PMIX_SCOPE_UNDEF; - cb.copy = false; - PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); - if (PMIX_SUCCESS == rc) { - PMIX_LIST_FOREACH(kv, &cb.kvs, pmix_kval_t) { - PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, kv, 1, PMIX_KVAL); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(reply); - PMIX_DESTRUCT(&cb); - goto cleanup; - } + /* pass back the name of the nspace */ + PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, &cd->pname.nspace, 1, PMIX_STRING); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + goto cleanup; + } + /* add the job-level info, if we have it */ + pmix_strncpy(proc.nspace, cd->pname.nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + /* this is going to a local client, so let the gds + * have the option of returning a copy of the data, + * or a pointer to local storage */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + cb.proc = &proc; + cb.scope = PMIX_SCOPE_UNDEF; + cb.copy = false; + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); + if (PMIX_SUCCESS == rc) { + PMIX_LIST_FOREACH(kv, &cb.kvs, pmix_kval_t) { + PMIX_BFROPS_PACK(rc, cd->cd->peer, reply, kv, 1, PMIX_KVAL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + PMIX_DESTRUCT(&cb); + goto cleanup; } - PMIX_DESTRUCT(&cb); } + PMIX_DESTRUCT(&cb); } /* the function that created the server_caddy did a * retain on the peer, so we don't have to worry about * it still being present - tell the originator the result */ - PMIX_SERVER_QUEUE_REPLY(cd->cd->peer, cd->cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->cd->peer, cd->cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } cleanup: /* cleanup */ @@ -2166,7 +2258,10 @@ static void lookup_cbfunc(pmix_status_t status, pmix_pdata_t pdata[], size_t nda /* the function that created the server_caddy did a * retain on the peer, so we don't have to worry about * it still being present - tell the originator the result */ - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } /* cleanup */ PMIX_RELEASE(cd); } @@ -2180,19 +2275,33 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) { pmix_shift_caddy_t *scd = (pmix_shift_caddy_t*)cbdata; pmix_server_trkr_t *tracker = scd->tracker; - pmix_buffer_t xfer, *reply, bkt; - pmix_byte_object_t bo, bo2; - pmix_server_caddy_t *cd; + pmix_buffer_t xfer, *reply; + pmix_server_caddy_t *cd, *nxt; pmix_status_t rc = PMIX_SUCCESS, ret; pmix_nspace_caddy_t *nptr; pmix_list_t nslist; - int32_t cnt = 1; - char byte; bool found; - pmix_collect_t ctype; PMIX_ACQUIRE_OBJECT(scd); + if (NULL == tracker) { + /* give them a release if they want it - this should + * never happen, but protect against the possibility */ + if (NULL != scd->cbfunc.relfn) { + scd->cbfunc.relfn(scd->cbdata); + } + PMIX_RELEASE(scd); + return; + } + + /* if we get here, then there are processes waiting + * for a response */ + + /* if the timer is active, clear it */ + if (tracker->event_active) { + pmix_event_del(&tracker->ev); + } + /* pass the blobs being returned */ PMIX_CONSTRUCT(&xfer, pmix_buffer_t); PMIX_LOAD_BUFFER(pmix_globals.mypeer, &xfer, scd->data, scd->ndata); @@ -2214,7 +2323,7 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) goto finish_collective; } - // collect the pmix_nspace_t's of all local participants + // collect the pmix_namespace_t's of all local participants PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { // see if we already have this nspace found = false; @@ -2233,74 +2342,17 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) } } - /* Loop over the enclosed byte object envelopes and - * store them in our GDS module */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &xfer, &bo, &cnt, PMIX_BYTE_OBJECT); - while (PMIX_SUCCESS == rc) { - PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, bo.bytes, bo.size); - /* unpack the data collection flag */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &byte, &cnt, PMIX_BYTE); - if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { - /* no data was returned, so we are done with this blob */ - break; - } + PMIX_LIST_FOREACH(nptr, &nslist, pmix_nspace_caddy_t) { + PMIX_GDS_STORE_MODEX(rc, nptr->ns, &tracker->local_cbs, &xfer); if (PMIX_SUCCESS != rc) { - /* we have an error */ - break; - } - - // Check that this blob was accumulated with the same data collection setting - ctype = (pmix_collect_t)byte; - if (ctype != tracker->collect_type) { - rc = PMIX_ERR_INVALID_ARG; - break; - } - /* unpack the enclosed blobs from the various peers */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &bo2, &cnt, PMIX_BYTE_OBJECT); - while (PMIX_SUCCESS == rc) { - /* unpack all the kval's from this peer and store them in - * our GDS. Note that PMIx by design holds all data at - * the server level until requested. If our GDS is a - * shared memory region, then the data may be available - * right away - but the client still has to be notified - * of its presence. */ - PMIX_LIST_FOREACH(nptr, &nslist, pmix_nspace_caddy_t) { - PMIX_GDS_STORE_MODEX(rc, nptr->ns, &tracker->local_cbs, &bo2); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - } - PMIX_BYTE_OBJECT_DESTRUCT(&bo2); - /* get the next blob */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &bo2, &cnt, PMIX_BYTE_OBJECT); - } - if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { - rc = PMIX_SUCCESS; - } else if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto finish_collective; + break; } - /* unpack and process the next blob */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &xfer, &bo, &cnt, PMIX_BYTE_OBJECT); - } - if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER == rc) { - rc = PMIX_SUCCESS; } finish_collective: /* loop across all procs in the tracker, sending them the reply */ - PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { + PMIX_LIST_FOREACH_SAFE(cd, nxt, &tracker->local_cbs, pmix_server_caddy_t) { reply = PMIX_NEW(pmix_buffer_t); if (NULL == reply) { rc = PMIX_ERR_NOMEM; @@ -2315,7 +2367,13 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) pmix_output_verbose(2, pmix_server_globals.base_output, "server:modex_cbfunc reply being sent to %s:%u", cd->peer->info->pname.nspace, cd->peer->info->pname.rank); - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + /* remove this entry */ + pmix_list_remove_item(&tracker->local_cbs, &cd->super); + PMIX_RELEASE(cd); } cleanup: @@ -2328,7 +2386,12 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) xfer.bytes_used = 0; PMIX_DESTRUCT(&xfer); - pmix_list_remove_item(&pmix_server_globals.collectives, &tracker->super); + if (!tracker->lost_connection) { + /* if this tracker has gone thru the "lost_connection" procedure, + * then it has already been removed from the list - otherwise, + * remove it now */ + pmix_list_remove_item(&pmix_server_globals.collectives, &tracker->super); + } PMIX_RELEASE(tracker); PMIX_LIST_DESTRUCT(&nslist); @@ -2348,15 +2411,6 @@ static void modex_cbfunc(pmix_status_t status, const char *data, size_t ndata, v pmix_output_verbose(2, pmix_server_globals.base_output, "server:modex_cbfunc called with %d bytes", (int)ndata); - if (NULL == tracker) { - /* nothing to do - but be sure to give them - * a release if they want it */ - if (NULL != relfn) { - relfn(relcbd); - } - return; - } - /* need to thread-shift this callback as it accesses global data */ scd = PMIX_NEW(pmix_shift_caddy_t); if (NULL == scd) { @@ -2385,7 +2439,10 @@ static void get_cbfunc(pmix_status_t status, const char *data, size_t ndata, voi pmix_output_verbose(2, pmix_server_globals.base_output, "server:get_cbfunc called with %d bytes", (int)ndata); - /* no need to thread-shift here as no global data is accessed */ + /* no need to thread-shift here as no global data is accessed + * and we are called from another internal function + * (see pmix_server_get.c:pmix_pending_resolve) that + * has already been thread-shifted */ if (NULL == cd) { /* nothing to do - but be sure to give them @@ -2421,7 +2478,10 @@ static void get_cbfunc(pmix_status_t status, const char *data, size_t ndata, voi pmix_output_hexdump(10, pmix_server_globals.base_output, reply->base_ptr, (reply->bytes_used < 256 ? reply->bytes_used : 256)); - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } cleanup: /* if someone wants a release, give it to them */ @@ -2448,6 +2508,19 @@ static void _cnct(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(scd); + if (NULL == tracker) { + /* nothing to do */ + return; + } + + /* if we get here, then there are processes waiting + * for a response */ + + /* if the timer is active, clear it */ + if (tracker->event_active) { + pmix_event_del(&tracker->ev); + } + /* find the unique nspaces that are participating */ PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { if (NULL == nspaces) { @@ -2497,7 +2570,7 @@ static void _cnct(int sd, short args, void *cbdata) * local storage */ /* add the job-level info, if necessary */ proc.rank = PMIX_RANK_WILDCARD; - (void)strncpy(proc.nspace, nspaces[i], PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspaces[i], PMIX_MAX_NSLEN); PMIX_CONSTRUCT(&cb, pmix_cb_t); /* this is for a local client, so give the gds the * option of returning a complete copy of the data, @@ -2561,14 +2634,22 @@ static void _cnct(int sd, short args, void *cbdata) pmix_output_verbose(2, pmix_server_globals.base_output, "server:cnct_cbfunc reply being sent to %s:%u", cd->peer->info->pname.nspace, cd->peer->info->pname.rank); - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } } cleanup: if (NULL != nspaces) { pmix_argv_free(nspaces); } - pmix_list_remove_item(&pmix_server_globals.collectives, &tracker->super); + if (!tracker->lost_connection) { + /* if this tracker has gone thru the "lost_connection" procedure, + * then it has already been removed from the list - otherwise, + * remove it now */ + pmix_list_remove_item(&pmix_server_globals.collectives, &tracker->super); + } PMIX_RELEASE(tracker); /* we are done */ @@ -2583,11 +2664,6 @@ static void cnct_cbfunc(pmix_status_t status, void *cbdata) pmix_output_verbose(2, pmix_server_globals.base_output, "server:cnct_cbfunc called"); - if (NULL == tracker) { - /* nothing to do */ - return; - } - /* need to thread-shift this callback as it accesses global data */ scd = PMIX_NEW(pmix_shift_caddy_t); if (NULL == scd) { @@ -2609,6 +2685,19 @@ static void _discnct(int sd, short args, void *cbdata) PMIX_ACQUIRE_OBJECT(scd); + if (NULL == tracker) { + /* nothing to do */ + return; + } + + /* if we get here, then there are processes waiting + * for a response */ + + /* if the timer is active, clear it */ + if (tracker->event_active) { + pmix_event_del(&tracker->ev); + } + /* loop across all local procs in the tracker, sending them the reply */ PMIX_LIST_FOREACH(cd, &tracker->local_cbs, pmix_server_caddy_t) { /* setup the reply */ @@ -2628,13 +2717,21 @@ static void _discnct(int sd, short args, void *cbdata) pmix_output_verbose(2, pmix_server_globals.base_output, "server:cnct_cbfunc reply being sent to %s:%u", cd->peer->info->pname.nspace, cd->peer->info->pname.rank); - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } } cleanup: /* cleanup the tracker -- the host RM is responsible for * telling us when to remove the nspace from our data */ - pmix_list_remove_item(&pmix_server_globals.collectives, &tracker->super); + if (!tracker->lost_connection) { + /* if this tracker has gone thru the "lost_connection" procedure, + * then it has already been removed from the list - otherwise, + * remove it now */ + pmix_list_remove_item(&pmix_server_globals.collectives, &tracker->super); + } PMIX_RELEASE(tracker); /* we are done */ @@ -2650,11 +2747,6 @@ static void discnct_cbfunc(pmix_status_t status, void *cbdata) "server:discnct_cbfunc called on nspace %s", (NULL == tracker) ? "NULL" : tracker->pname.nspace); - if (NULL == tracker) { - /* nothing to do */ - return; - } - /* need to thread-shift this callback as it accesses global data */ scd = PMIX_NEW(pmix_shift_caddy_t); if (NULL == scd) { @@ -2687,7 +2779,10 @@ static void regevents_cbfunc(pmix_status_t status, void *cbdata) PMIX_ERROR_LOG(rc); } // send reply - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } PMIX_RELEASE(cd); } @@ -2711,10 +2806,71 @@ static void notifyerror_cbfunc (pmix_status_t status, void *cbdata) PMIX_ERROR_LOG(rc); } // send reply - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } PMIX_RELEASE(cd); } +static void alloc_cbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata; + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata; + pmix_buffer_t *reply; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:alloc callback with status %d", status); + + reply = PMIX_NEW(pmix_buffer_t); + if (NULL == reply) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + PMIX_RELEASE(cd); + return; + } + PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + /* pack the returned data */ + PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + if (0 < ninfo) { + PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } + + complete: + // send reply + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + + // cleanup + if (NULL != qcd->queries) { + PMIX_QUERY_FREE(qcd->queries, qcd->nqueries); + } + if (NULL != qcd->info) { + PMIX_INFO_FREE(qcd->info, qcd->ninfo); + } + PMIX_RELEASE(qcd); + PMIX_RELEASE(cd); + if (NULL != release_fn) { + release_fn(release_cbdata); + } +} static void query_cbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, @@ -2754,9 +2910,15 @@ static void query_cbfunc(pmix_status_t status, } } + /* cache the data for any future requests */ + complete: // send reply - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + // cleanup if (NULL != qcd->queries) { PMIX_QUERY_FREE(qcd->queries, qcd->nqueries); @@ -2766,6 +2928,127 @@ static void query_cbfunc(pmix_status_t status, } PMIX_RELEASE(qcd); PMIX_RELEASE(cd); + if (NULL != release_fn) { + release_fn(release_cbdata); + } +} + +static void jctrl_cbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata; + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata; + pmix_buffer_t *reply; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:jctrl callback with status %d", status); + + reply = PMIX_NEW(pmix_buffer_t); + if (NULL == reply) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + PMIX_RELEASE(cd); + return; + } + PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + /* pack the returned data */ + PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + if (0 < ninfo) { + PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } + + complete: + // send reply + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + + // cleanup + if (NULL != qcd->queries) { + PMIX_QUERY_FREE(qcd->queries, qcd->nqueries); + } + if (NULL != qcd->info) { + PMIX_INFO_FREE(qcd->info, qcd->ninfo); + } + PMIX_RELEASE(qcd); + PMIX_RELEASE(cd); + if (NULL != release_fn) { + release_fn(release_cbdata); + } +} + +static void monitor_cbfunc(pmix_status_t status, + pmix_info_t *info, size_t ninfo, + void *cbdata, + pmix_release_cbfunc_t release_fn, + void *release_cbdata) +{ + pmix_query_caddy_t *qcd = (pmix_query_caddy_t*)cbdata; + pmix_server_caddy_t *cd = (pmix_server_caddy_t*)qcd->cbdata; + pmix_buffer_t *reply; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_server_globals.base_output, + "pmix:monitor callback with status %d", status); + + reply = PMIX_NEW(pmix_buffer_t); + if (NULL == reply) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + PMIX_RELEASE(cd); + return; + } + PMIX_BFROPS_PACK(rc, cd->peer, reply, &status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + /* pack the returned data */ + PMIX_BFROPS_PACK(rc, cd->peer, reply, &ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + goto complete; + } + if (0 < ninfo) { + PMIX_BFROPS_PACK(rc, cd->peer, reply, info, ninfo, PMIX_INFO); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } + + complete: + // send reply + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + + // cleanup + if (NULL != qcd->queries) { + PMIX_QUERY_FREE(qcd->queries, qcd->nqueries); + } + if (NULL != qcd->info) { + PMIX_INFO_FREE(qcd->info, qcd->ninfo); + } + PMIX_RELEASE(qcd); + PMIX_RELEASE(cd); + if (NULL != release_fn) { + release_fn(release_cbdata); + } } static void cred_cbfunc(pmix_status_t status, @@ -2819,7 +3102,11 @@ static void cred_cbfunc(pmix_status_t status, complete: // send reply - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } + // cleanup if (NULL != qcd->info) { PMIX_INFO_FREE(qcd->info, qcd->ninfo); @@ -2866,7 +3153,10 @@ static void validate_cbfunc(pmix_status_t status, complete: // send reply - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } // cleanup if (NULL != qcd->info) { PMIX_INFO_FREE(qcd->info, qcd->ninfo); @@ -2908,7 +3198,10 @@ static void _iofreg(int sd, short args, void *cbdata) pmix_output_verbose(2, pmix_server_globals.iof_output, "server:_iofreg reply being sent to %s:%u", scd->peer->info->pname.nspace, scd->peer->info->pname.rank); - PMIX_SERVER_QUEUE_REPLY(scd->peer, scd->hdr.tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, scd->peer, scd->hdr.tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } cleanup: /* release the cached info */ @@ -2965,8 +3258,6 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, pmix_server_caddy_t *cd; pmix_proc_t proc; pmix_buffer_t *reply; - pmix_regevents_info_t *reginfo; - pmix_peer_events_info_t *prev; /* retrieve the cmd */ cnt = 1; @@ -2976,8 +3267,8 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, return rc; } pmix_output_verbose(2, pmix_server_globals.base_output, - "recvd pmix cmd %d from %s:%u", - cmd, peer->info->pname.nspace, peer->info->pname.rank); + "recvd pmix cmd %s from %s:%u", + pmix_command_string(cmd), peer->info->pname.nspace, peer->info->pname.rank); if (PMIX_REQ_CMD == cmd) { reply = PMIX_NEW(pmix_buffer_t); @@ -2990,7 +3281,10 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, PMIX_ERROR_LOG(rc); return rc; } - PMIX_SERVER_QUEUE_REPLY(peer, tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, peer, tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } peer->nptr->ndelivered++; return PMIX_SUCCESS; } @@ -3015,7 +3309,10 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } - PMIX_SERVER_QUEUE_REPLY(peer, tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, peer, tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } } return PMIX_SUCCESS; // don't reply twice } @@ -3039,20 +3336,9 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, if (PMIX_FINALIZE_CMD == cmd) { pmix_output_verbose(2, pmix_server_globals.base_output, "recvd FINALIZE"); - /* mark that this peer called finalize */ - peer->finalized = true; peer->nptr->nfinalized++; - /* since the client is finalizing, remove them from any event - * registrations they may still have on our list */ - PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { - PMIX_LIST_FOREACH(prev, ®info->peers, pmix_peer_events_info_t) { - if (prev->peer == peer) { - pmix_list_remove_item(®info->peers, &prev->super); - PMIX_RELEASE(prev); - break; - } - } - } + /* purge events */ + pmix_server_purge_events(peer, NULL); /* turn off the recv event - we shouldn't hear anything * more from this proc */ if (peer->recv_ev_active) { @@ -3062,14 +3348,18 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, PMIX_GDS_CADDY(cd, peer, tag); /* call the local server, if supported */ if (NULL != pmix_host_server.client_finalized) { - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; /* now tell the host server */ - if (PMIX_SUCCESS == (rc = pmix_host_server.client_finalized(&proc, peer->info->server_object, - op_cbfunc2, cd))) { + rc = pmix_host_server.client_finalized(&proc, peer->info->server_object, + op_cbfunc2, cd); + if (PMIX_SUCCESS == rc) { /* don't reply to them ourselves - we will do so when the host * server calls us back */ return rc; + } else if (PMIX_OPERATION_SUCCEEDED == rc) { + /* they did it atomically */ + rc = PMIX_SUCCESS; } /* if the call doesn't succeed (e.g., they provided the stub * but return NOT_SUPPORTED), then the callback function @@ -3077,7 +3367,7 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, * any lingering references to this peer and answer * the client. Thus, we call the callback function ourselves * in this case */ - op_cbfunc2(PMIX_SUCCESS, cd); + op_cbfunc2(rc, cd); /* return SUCCESS as the cbfunc generated the return msg * and released the cd object */ return PMIX_SUCCESS; @@ -3134,14 +3424,18 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, if (PMIX_CONNECTNB_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); rc = pmix_server_connect(cd, buf, cnct_cbfunc); - PMIX_RELEASE(cd); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_DISCONNECTNB_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); rc = pmix_server_disconnect(cd, buf, discnct_cbfunc); - PMIX_RELEASE(cd); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cd); + } return rc; } @@ -3160,61 +3454,81 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, if (PMIX_NOTIFY_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_event_recvd_from_client(peer, buf, notifyerror_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_event_recvd_from_client(peer, buf, notifyerror_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_QUERY_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_query(peer, buf, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_query(peer, buf, query_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_LOG_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_log(peer, buf, op_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_log(peer, buf, op_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_ALLOC_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_alloc(peer, buf, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_alloc(peer, buf, alloc_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_JOB_CONTROL_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_job_ctrl(peer, buf, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_job_ctrl(peer, buf, jctrl_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_MONITOR_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_monitor(peer, buf, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_monitor(peer, buf, monitor_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_GET_CREDENTIAL_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_get_credential(peer, buf, cred_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_get_credential(peer, buf, cred_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_VALIDATE_CRED_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_validate_credential(peer, buf, validate_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_validate_credential(peer, buf, validate_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_IOF_PULL_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_iofreg(peer, buf, iof_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_iofreg(peer, buf, iof_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } if (PMIX_IOF_PUSH_CMD == cmd) { PMIX_GDS_CADDY(cd, peer, tag); - rc = pmix_server_iofstdin(peer, buf, op_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_server_iofstdin(peer, buf, op_cbfunc, cd))) { + PMIX_RELEASE(cd); + } return rc; } @@ -3249,6 +3563,9 @@ void pmix_server_message_handler(struct pmix_peer_t *pr, if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); } - PMIX_SERVER_QUEUE_REPLY(peer, hdr->tag, reply); + PMIX_SERVER_QUEUE_REPLY(rc, peer, hdr->tag, reply); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(reply); + } } } diff --git a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c index a0e474e596..56c0530801 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c +++ b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_get.c @@ -84,7 +84,7 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_reply_caddy_t, static void dmdx_cbfunc(pmix_status_t status, const char *data, size_t ndata, void *cbdata, pmix_release_cbfunc_t relfn, void *relcbdata); -static pmix_status_t _satisfy_request(pmix_nspace_t *ns, pmix_rank_t rank, +static pmix_status_t _satisfy_request(pmix_namespace_t *ns, pmix_rank_t rank, pmix_server_caddy_t *cd, pmix_modex_cbfunc_t cbfunc, void *cbdata, bool *scope); static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank, @@ -119,7 +119,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, pmix_rank_t rank; char *cptr; char nspace[PMIX_MAX_NSLEN+1]; - pmix_nspace_t *ns, *nptr; + pmix_namespace_t *ns, *nptr; pmix_info_t *info=NULL; size_t ninfo=0; pmix_dmdx_local_t *lcd; @@ -148,7 +148,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, PMIX_ERROR_LOG(rc); return rc; } - (void)strncpy(nspace, cptr, PMIX_MAX_NSLEN); + pmix_strncpy(nspace, cptr, PMIX_MAX_NSLEN); free(cptr); cnt = 1; PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &rank, &cnt, PMIX_PROC_RANK); @@ -191,7 +191,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, /* find the nspace object for this client */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(nspace, ns->nspace)) { nptr = ns; break; @@ -245,14 +245,22 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, if (PMIX_ERR_NOMEM == rc) { PMIX_INFO_FREE(info, ninfo); return rc; - } else if (PMIX_ERR_NOT_FOUND != rc) { - return rc; } - - /* do NOT create the nspace tracker here so any request - * by another local client that hits before the RM responds - * to our request will get added to the local tracker so - * they receive their data upon completion */ + if (PMIX_SUCCESS == rc) { + /* if they specified a timeout for this specific + * request, set it up now */ + if (0 < tv.tv_sec) { + pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, + get_timeout, req); + pmix_event_evtimer_add(&req->ev, &tv); + req->event_active = true; + } + /* we already asked for this info - no need to + * do it again */ + return PMIX_SUCCESS; + } + /* only other return code is NOT_FOUND, indicating that + * we created a new tracker */ /* Its possible there will be no local processes on this * host, so lets ask for this explicitly. There can @@ -260,16 +268,29 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * up on its own, but at worst the direct modex * will simply overwrite the info later */ if (NULL != pmix_host_server.direct_modex) { - pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd); + rc = pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd); + if (PMIX_SUCCESS != rc) { + PMIX_INFO_FREE(info, ninfo); + pmix_list_remove_item(&pmix_server_globals.local_reqs, &lcd->super); + PMIX_RELEASE(lcd); + return rc; + } + /* if they specified a timeout for this specific + * request, set it up now */ + if (0 < tv.tv_sec) { + pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, + get_timeout, req); + pmix_event_evtimer_add(&req->ev, &tv); + req->event_active = true; + } + } else { + /* if we don't have direct modex feature, just respond with "not found" */ + PMIX_INFO_FREE(info, ninfo); + pmix_list_remove_item(&pmix_server_globals.local_reqs, &lcd->super); + PMIX_RELEASE(lcd); + return PMIX_ERR_NOT_FOUND; } - /* if they specified a timeout, set it up now */ - if (0 < tv.tv_sec) { - pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, - get_timeout, req); - pmix_event_evtimer_add(&req->ev, &tv); - req->event_active = true; - } return PMIX_SUCCESS; } @@ -282,7 +303,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * for it, so there is no guarantee we have it */ data = NULL; sz = 0; - (void)strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; /* if we have local procs for this nspace, then we * can retrieve the info from that GDS. Otherwise, @@ -323,7 +344,11 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, /* unload the resulting payload */ PMIX_UNLOAD_BUFFER(&pbkt, data, sz); PMIX_DESTRUCT(&pbkt); + /* call the internal callback function - it will + * release the cbdata */ cbfunc(PMIX_SUCCESS, data, sz, cbdata, relfn, data); + /* return success so the server doesn't duplicate + * the release of cbdata */ return PMIX_SUCCESS; } @@ -353,6 +378,7 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, cbfunc, cbdata, &lcd, &req); if (PMIX_ERR_NOMEM == rc) { PMIX_INFO_FREE(info, ninfo); + return rc; } pmix_output_verbose(2, pmix_server_globals.get_output, "%s:%d TRACKER CREATED - WAITING", @@ -365,15 +391,20 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, pmix_event_evtimer_add(&req->ev, &tv); req->event_active = true; } - return rc; + /* the peer object has been added to the new lcd tracker, + * so return success here */ + return PMIX_SUCCESS; } - /* see if we already have this data */ + /* if everyone has registered, see if we already have this data */ rc = _satisfy_request(nptr, rank, cd, cbfunc, cbdata, &local); if( PMIX_SUCCESS == rc ){ /* request was successfully satisfied */ PMIX_INFO_FREE(info, ninfo); - return rc; + /* return success as the satisfy_request function + * calls the cbfunc for us, and it will have + * released the cbdata object */ + return PMIX_SUCCESS; } pmix_output_verbose(2, pmix_server_globals.get_output, @@ -395,18 +426,24 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * we do, then we can just wait for it to arrive */ rc = create_local_tracker(nspace, rank, info, ninfo, cbfunc, cbdata, &lcd, &req); + if (PMIX_ERR_NOMEM == rc || NULL == lcd) { + /* we have a problem */ + PMIX_INFO_FREE(info, ninfo); + return PMIX_ERR_NOMEM; + } + /* if they specified a timeout, set it up now */ + if (0 < tv.tv_sec) { + pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, + get_timeout, req); + pmix_event_evtimer_add(&req->ev, &tv); + req->event_active = true; + } if (PMIX_SUCCESS == rc) { /* we are already waiting for the data - nothing more * for us to do as the function added the new request * to the tracker for us */ return PMIX_SUCCESS; } - if (PMIX_ERR_NOT_FOUND != rc || NULL == lcd) { - /* we have a problem - e.g., out of memory */ - cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL); - PMIX_INFO_FREE(info, ninfo); - return rc; - } /* Getting here means that we didn't already have a request for * for data pending, and so we created a new tracker for this @@ -414,13 +451,6 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * if this is one, then we have nothing further to do - we will * fulfill the request once the process commits its data */ if (local) { - /* if they specified a timeout, set it up now */ - if (0 < tv.tv_sec) { - pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, - get_timeout, req); - pmix_event_evtimer_add(&req->ev, &tv); - req->event_active = true; - } return PMIX_SUCCESS; } @@ -429,12 +459,11 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, * whomever is hosting the target process */ if (NULL != pmix_host_server.direct_modex) { rc = pmix_host_server.direct_modex(&lcd->proc, info, ninfo, dmdx_cbfunc, lcd); - /* if they specified a timeout, set it up now */ - if (0 < tv.tv_sec) { - pmix_event_evtimer_set(pmix_globals.evbase, &req->ev, - get_timeout, req); - pmix_event_evtimer_add(&req->ev, &tv); - req->event_active = true; + if (PMIX_SUCCESS != rc) { + /* may have a function entry but not support the request */ + PMIX_INFO_FREE(info, ninfo); + pmix_list_remove_item(&pmix_server_globals.local_reqs, &lcd->super); + PMIX_RELEASE(lcd); } } else { pmix_output_verbose(2, pmix_server_globals.get_output, @@ -442,7 +471,6 @@ pmix_status_t pmix_server_get(pmix_buffer_t *buf, pmix_globals.myid.nspace, pmix_globals.myid.rank); /* if we don't have direct modex feature, just respond with "not found" */ - cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL); PMIX_INFO_FREE(info, ninfo); pmix_list_remove_item(&pmix_server_globals.local_reqs, &lcd->super); PMIX_RELEASE(lcd); @@ -490,7 +518,7 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank, if (NULL == lcd){ return PMIX_ERR_NOMEM; } - strncpy(lcd->proc.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(lcd->proc.nspace, nspace, PMIX_MAX_NSLEN); lcd->proc.rank = rank; lcd->info = info; lcd->ninfo = ninfo; @@ -515,9 +543,10 @@ static pmix_status_t create_local_tracker(char nspace[], pmix_rank_t rank, return rc; } -void pmix_pending_nspace_requests(pmix_nspace_t *nptr) +void pmix_pending_nspace_requests(pmix_namespace_t *nptr) { pmix_dmdx_local_t *cd, *cd_next; + pmix_status_t rc; /* Now that we know all local ranks, go along request list and ask for remote data * for the non-local ranks, and resolve all pending requests for local procs @@ -540,10 +569,12 @@ void pmix_pending_nspace_requests(pmix_nspace_t *nptr) /* if not found - this is remote process and we need to send * corresponding direct modex request */ - if( !found ){ - if( NULL != pmix_host_server.direct_modex ){ - pmix_host_server.direct_modex(&cd->proc, cd->info, cd->ninfo, dmdx_cbfunc, cd); - } else { + if (!found){ + rc = PMIX_ERR_NOT_SUPPORTED; + if (NULL != pmix_host_server.direct_modex){ + rc = pmix_host_server.direct_modex(&cd->proc, cd->info, cd->ninfo, dmdx_cbfunc, cd); + } + if (PMIX_SUCCESS != rc) { pmix_dmdx_request_t *req, *req_next; PMIX_LIST_FOREACH_SAFE(req, req_next, &cd->loc_reqs, pmix_dmdx_request_t) { req->cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, req->cbdata, NULL, NULL); @@ -557,7 +588,7 @@ void pmix_pending_nspace_requests(pmix_nspace_t *nptr) } } -static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, +static pmix_status_t _satisfy_request(pmix_namespace_t *nptr, pmix_rank_t rank, pmix_server_caddy_t *cd, pmix_modex_cbfunc_t cbfunc, void *cbdata, bool *local) @@ -584,7 +615,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, * a remote peer, or due to data from a local client * having been committed */ PMIX_CONSTRUCT(&pbkt, pmix_buffer_t); - (void)strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); /* if we have local clients of this nspace, then we use * the corresponding GDS to retrieve the data. Otherwise, @@ -771,7 +802,7 @@ static pmix_status_t _satisfy_request(pmix_nspace_t *nptr, pmix_rank_t rank, } /* Resolve pending requests to this namespace/rank */ -pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, +pmix_status_t pmix_pending_resolve(pmix_namespace_t *nptr, pmix_rank_t rank, pmix_status_t status, pmix_dmdx_local_t *lcd) { pmix_dmdx_local_t *cd, *ptr; @@ -783,7 +814,7 @@ pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, ptr = NULL; if (NULL != nptr) { PMIX_LIST_FOREACH(cd, &pmix_server_globals.local_reqs, pmix_dmdx_local_t) { - if (0 != strncmp(nptr->nspace, cd->proc.nspace, PMIX_MAX_NSLEN) || + if (!PMIX_CHECK_NSPACE(nptr->nspace, cd->proc.nspace) || rank != cd->proc.rank) { continue; } @@ -798,6 +829,13 @@ pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, ptr = lcd; } + /* if there are no local reqs on this request (e.g., only + * one proc requested it and that proc has died), then + * just remove the request */ + if (0 == pmix_list_get_size(&ptr->loc_reqs)) { + goto cleanup; + } + /* somebody was interested in this rank */ if (PMIX_SUCCESS != status){ /* if we've got an error for this request - just forward it*/ @@ -822,8 +860,10 @@ pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, } PMIX_RELEASE(scd); } + + cleanup: /* remove all requests to this rank and cleanup the corresponding structure */ - pmix_list_remove_item(&pmix_server_globals.local_reqs, (pmix_list_item_t*)ptr); + pmix_list_remove_item(&pmix_server_globals.local_reqs, &ptr->super); PMIX_RELEASE(ptr); return PMIX_SUCCESS; @@ -838,7 +878,7 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata) pmix_rank_info_t *rinfo; int32_t cnt; pmix_kval_t *kv; - pmix_nspace_t *ns, *nptr; + pmix_namespace_t *ns, *nptr; pmix_status_t rc; pmix_list_t nspaces; pmix_nspace_caddy_t *nm; @@ -856,7 +896,7 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata) /* find the nspace object for the proc whose data is being received */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(caddy->lcd->proc.nspace, ns->nspace)) { nptr = ns; break; @@ -867,7 +907,7 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata) /* We may not have this namespace because there are no local * processes from it running on this host - so just record it * so we know we have the data for any future requests */ - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); nptr->nspace = strdup(caddy->lcd->proc.nspace); /* add to the list */ pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); @@ -931,7 +971,7 @@ static void _process_dmdx_reply(int fd, short args, void *cbdata) PMIX_DESTRUCT(&cb); goto complete; } - (void)strncpy(cb.proc->nspace, nm->ns->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(cb.proc->nspace, nm->ns->nspace, PMIX_MAX_NSLEN); cb.proc->rank = PMIX_RANK_WILDCARD; cb.scope = PMIX_INTERNAL; cb.copy = false; diff --git a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c index 7293c6e155..5e239e0d03 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c +++ b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.c @@ -1,8 +1,8 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2014-2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2016-2017 Mellanox Technologies, Inc. @@ -46,6 +46,9 @@ #ifdef HAVE_SYS_TYPES_H #include #endif +#ifdef HAVE_TIME_H +#include +#endif #include PMIX_EVENT_HEADER #include "src/class/pmix_hotel.h" @@ -116,16 +119,12 @@ pmix_status_t pmix_server_abort(pmix_peer_t *peer, pmix_buffer_t *buf, /* let the local host's server execute it */ if (NULL != pmix_host_server.abort) { - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.abort(&proc, peer->info->server_object, status, msg, procs, nprocs, cbfunc, cbdata); } else { rc = PMIX_ERR_NOT_SUPPORTED; - /* release the caller */ - if (NULL != cbfunc) { - cbfunc(rc, cbdata); - } } PMIX_PROC_FREE(procs, nprocs); @@ -145,7 +144,7 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf) pmix_buffer_t b2, pbkt; pmix_kval_t *kp; pmix_scope_t scope; - pmix_nspace_t *nptr; + pmix_namespace_t *nptr; pmix_rank_info_t *info; pmix_proc_t proc; pmix_dmdx_remote_t *dcd, *dcdnext; @@ -156,7 +155,7 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf) /* shorthand */ info = peer->info; nptr = peer->nptr; - (void)strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); proc.rank = info->pname.rank; pmix_output_verbose(2, pmix_server_globals.base_output, @@ -293,7 +292,7 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf) * regardless of location * nprocs - the number of procs in the array */ -static pmix_server_trkr_t* get_tracker(pmix_proc_t *procs, +static pmix_server_trkr_t* get_tracker(char *id, pmix_proc_t *procs, size_t nprocs, pmix_cmd_t type) { pmix_server_trkr_t *trk; @@ -304,7 +303,7 @@ static pmix_server_trkr_t* get_tracker(pmix_proc_t *procs, "get_tracker called with %d procs", (int)nprocs); /* bozo check - should never happen outside of programmer error */ - if (NULL == procs) { + if (NULL == procs && NULL == id) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); return NULL; } @@ -317,28 +316,35 @@ static pmix_server_trkr_t* get_tracker(pmix_proc_t *procs, * shouldn't take long */ PMIX_LIST_FOREACH(trk, &pmix_server_globals.collectives, pmix_server_trkr_t) { /* Collective operation if unique identified by - * the set of participating processes and the type of collective + * the set of participating processes and the type of collective, + * or by the operation ID */ - if (nprocs != trk->npcs) { - continue; - } - if (type != trk->type) { - continue; - } - matches = 0; - for (i=0; i < nprocs; i++) { - /* the procs may be in different order, so we have - * to do an exhaustive search */ - for (j=0; j < trk->npcs; j++) { - if (0 == strcmp(procs[i].nspace, trk->pcs[j].nspace) && - procs[i].rank == trk->pcs[j].rank) { - ++matches; - break; + if (NULL != id) { + if (NULL != trk->id && 0 == strcmp(id, trk->id)) { + return trk; + } + } else { + if (nprocs != trk->npcs) { + continue; + } + if (type != trk->type) { + continue; + } + matches = 0; + for (i=0; i < nprocs; i++) { + /* the procs may be in different order, so we have + * to do an exhaustive search */ + for (j=0; j < trk->npcs; j++) { + if (0 == strcmp(procs[i].nspace, trk->pcs[j].nspace) && + procs[i].rank == trk->pcs[j].rank) { + ++matches; + break; + } } } - } - if (trk->npcs == matches) { - return trk; + if (trk->npcs == matches) { + return trk; + } } } /* No tracker was found */ @@ -361,13 +367,13 @@ static pmix_server_trkr_t* get_tracker(pmix_proc_t *procs, * regardless of location * nprocs - the number of procs in the array */ -static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, +static pmix_server_trkr_t* new_tracker(char *id, pmix_proc_t *procs, size_t nprocs, pmix_cmd_t type) { pmix_server_trkr_t *trk; size_t i; bool all_def; - pmix_nspace_t *nptr, *ns; + pmix_namespace_t *nptr, *ns; pmix_rank_info_t *info; pmix_output_verbose(5, pmix_server_globals.base_output, @@ -380,7 +386,8 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, } pmix_output_verbose(5, pmix_server_globals.base_output, - "adding new tracker with %d procs", (int)nprocs); + "adding new tracker %s with %d procs", + (NULL == id) ? "NO-ID" : id, (int)nprocs); /* this tracker is new - create it */ trk = PMIX_NEW(pmix_server_trkr_t); @@ -389,26 +396,35 @@ static pmix_server_trkr_t* new_tracker(pmix_proc_t *procs, return NULL; } - /* copy the procs */ - PMIX_PROC_CREATE(trk->pcs, nprocs); - if (NULL == trk->pcs) { - PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - PMIX_RELEASE(trk); - return NULL; + if (NULL != id) { + trk->id = strdup(id); + } + + if (NULL != procs) { + /* copy the procs */ + PMIX_PROC_CREATE(trk->pcs, nprocs); + if (NULL == trk->pcs) { + PMIX_ERROR_LOG(PMIX_ERR_NOMEM); + PMIX_RELEASE(trk); + return NULL; + } + memcpy(trk->pcs, procs, nprocs * sizeof(pmix_proc_t)); + trk->npcs = nprocs; } - trk->npcs = nprocs; trk->type = type; all_def = true; for (i=0; i < nprocs; i++) { - (void)strncpy(trk->pcs[i].nspace, procs[i].nspace, PMIX_MAX_NSLEN); - trk->pcs[i].rank = procs[i].rank; + if (NULL == id) { + pmix_strncpy(trk->pcs[i].nspace, procs[i].nspace, PMIX_MAX_NSLEN); + trk->pcs[i].rank = procs[i].rank; + } if (!all_def) { continue; } /* is this nspace known to us? */ nptr = NULL; - PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(procs[i].nspace, ns->nspace)) { nptr = ns; break; @@ -560,9 +576,9 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, } /* find/create the local tracker for this operation */ - if (NULL == (trk = get_tracker(procs, nprocs, PMIX_FENCENB_CMD))) { + if (NULL == (trk = get_tracker(NULL, procs, nprocs, PMIX_FENCENB_CMD))) { /* If no tracker was found - create and initialize it once */ - if (NULL == (trk = new_tracker(procs, nprocs, PMIX_FENCENB_CMD))) { + if (NULL == (trk = new_tracker(NULL, procs, nprocs, PMIX_FENCENB_CMD))) { /* only if a bozo error occurs */ PMIX_ERROR_LOG(PMIX_ERROR); /* DO NOT HANG */ @@ -650,7 +666,7 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, PMIX_LIST_FOREACH(scd, &trk->local_cbs, pmix_server_caddy_t) { /* get any remote contribution - note that there * may not be a contribution */ - (void)strncpy(pcs.nspace, scd->peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(pcs.nspace, scd->peer->info->pname.nspace, PMIX_MAX_NSLEN); pcs.rank = scd->peer->info->pname.rank; PMIX_CONSTRUCT(&cb, pmix_cb_t); cb.proc = &pcs; @@ -709,9 +725,13 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, /* now unload the blob and pass it upstairs */ PMIX_UNLOAD_BUFFER(&bucket, data, sz); PMIX_DESTRUCT(&bucket); - pmix_host_server.fence_nb(trk->pcs, trk->npcs, - trk->info, trk->ninfo, - data, sz, trk->modexcbfunc, trk); + rc = pmix_host_server.fence_nb(trk->pcs, trk->npcs, + trk->info, trk->ninfo, + data, sz, trk->modexcbfunc, trk); + if (PMIX_SUCCESS != rc) { + pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super); + PMIX_RELEASE(trk); + } } cleanup: @@ -792,12 +812,12 @@ pmix_status_t pmix_server_publish(pmix_peer_t *peer, goto cleanup; } } - (void)strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); + pmix_strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); cd->info[cd->ninfo-1].value.type = PMIX_UINT32; cd->info[cd->ninfo-1].value.data.uint32 = uid; /* call the local server */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.publish(&proc, cd->info, cd->ninfo, opcbfunc, cd); @@ -906,12 +926,12 @@ pmix_status_t pmix_server_lookup(pmix_peer_t *peer, goto cleanup; } } - (void)strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); + pmix_strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); cd->info[cd->ninfo-1].value.type = PMIX_UINT32; cd->info[cd->ninfo-1].value.data.uint32 = uid; /* call the local server */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.lookup(&proc, cd->keys, cd->info, cd->ninfo, lkcbfunc, cd); @@ -1002,12 +1022,12 @@ pmix_status_t pmix_server_unpublish(pmix_peer_t *peer, goto cleanup; } } - (void)strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); + pmix_strncpy(cd->info[cd->ninfo-1].key, PMIX_USERID, PMIX_MAX_KEYLEN); cd->info[cd->ninfo-1].value.type = PMIX_UINT32; cd->info[cd->ninfo-1].value.data.uint32 = uid; /* call the local server */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.unpublish(&proc, cd->keys, cd->info, cd->ninfo, opcbfunc, cd); @@ -1029,88 +1049,90 @@ static void spcbfunc(pmix_status_t status, { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; pmix_iof_req_t *req; - pmix_setup_caddy_t *occupant; - int i; pmix_buffer_t *msg; pmix_status_t rc; + pmix_iof_cache_t *iof, *ionext; /* if it was successful, and there are IOF requests, then * register them now */ if (PMIX_SUCCESS == status && PMIX_FWD_NO_CHANNELS != cd->channels) { /* record the request */ req = PMIX_NEW(pmix_iof_req_t); - if (NULL != req) { - PMIX_RETAIN(cd->peer); - req->peer = cd->peer; - req->pname.nspace = strdup(nspace); - req->pname.rank = PMIX_RANK_WILDCARD; - req->channels = cd->channels; - pmix_list_append(&pmix_globals.iof_requests, &req->super); + if (NULL == req) { + status = PMIX_ERR_NOMEM; + goto cleanup; } + PMIX_RETAIN(cd->peer); + req->peer = cd->peer; + req->pname.nspace = strdup(nspace); + req->pname.rank = PMIX_RANK_WILDCARD; + req->channels = cd->channels; + pmix_list_append(&pmix_globals.iof_requests, &req->super); /* process any cached IO */ - for (i=0; i < PMIX_IOF_HOTEL_SIZE; i++) { - pmix_hotel_knock(&pmix_server_globals.iof, PMIX_IOF_HOTEL_SIZE-i-1, (void**)&occupant); - if (NULL != occupant) { - if (!(occupant->channels & req->channels)) { - continue; - } - /* if the source matches the request, then forward this along */ - if (0 != strncmp(occupant->procs->nspace, req->pname.nspace, PMIX_MAX_NSLEN) || - (PMIX_RANK_WILDCARD != req->pname.rank && occupant->procs->rank != req->pname.rank)) { - continue; - } - /* never forward back to the source! This can happen if the source - * is a launcher */ - if (0 == strncmp(occupant->procs->nspace, req->peer->info->pname.nspace, PMIX_MAX_NSLEN) && - occupant->procs->rank == req->peer->info->pname.rank) { - continue; - } - /* setup the msg */ - if (NULL == (msg = PMIX_NEW(pmix_buffer_t))) { - PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - rc = PMIX_ERR_OUT_OF_RESOURCE; - break; - } - /* provide the source */ - PMIX_BFROPS_PACK(rc, req->peer, msg, occupant->procs, 1, PMIX_PROC); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - break; - } - /* provide the channel */ - PMIX_BFROPS_PACK(rc, req->peer, msg, &occupant->channels, 1, PMIX_IOF_CHANNEL); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - break; - } - /* pack the data */ - PMIX_BFROPS_PACK(rc, req->peer, msg, occupant->bo, 1, PMIX_BYTE_OBJECT); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - break; - } - /* send it to the requestor */ - PMIX_PTL_SEND_ONEWAY(rc, req->peer, msg, PMIX_PTL_TAG_IOF); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - } - /* remove it from the hotel since it has now been forwarded */ - pmix_hotel_checkout(&pmix_server_globals.iof, PMIX_IOF_HOTEL_SIZE-i-1); - PMIX_RELEASE(occupant); + PMIX_LIST_FOREACH_SAFE(iof, ionext, &pmix_server_globals.iof, pmix_iof_cache_t) { + /* if the channels don't match, then ignore it */ + if (!(iof->channel & req->channels)) { + continue; } + /* if the source does not match the request, then ignore it */ + if (!PMIX_CHECK_PROCID(&iof->source, &req->pname)) { + continue; + } + /* never forward back to the source! This can happen if the source + * is a launcher */ + if (PMIX_CHECK_PROCID(&iof->source, &req->peer->info->pname)) { + continue; + } + pmix_output_verbose(2, pmix_server_globals.iof_output, + "PMIX:SERVER:SPAWN delivering cached IOF from %s:%d to %s:%d", + iof->source.nspace, iof->source.rank, + req->pname.nspace, req->pname.rank); + /* setup the msg */ + if (NULL == (msg = PMIX_NEW(pmix_buffer_t))) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + rc = PMIX_ERR_OUT_OF_RESOURCE; + break; + } + /* provide the source */ + PMIX_BFROPS_PACK(rc, req->peer, msg, &iof->source, 1, PMIX_PROC); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + break; + } + /* provide the channel */ + PMIX_BFROPS_PACK(rc, req->peer, msg, &iof->channel, 1, PMIX_IOF_CHANNEL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + break; + } + /* pack the data */ + PMIX_BFROPS_PACK(rc, req->peer, msg, iof->bo, 1, PMIX_BYTE_OBJECT); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + break; + } + /* send it to the requestor */ + PMIX_PTL_SEND_ONEWAY(rc, req->peer, msg, PMIX_PTL_TAG_IOF); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + } + /* remove it from the list since it has now been forwarded */ + pmix_list_remove_item(&pmix_server_globals.iof, &iof->super); + PMIX_RELEASE(iof); } } + cleanup: /* cleanup the caddy */ if (NULL != cd->info) { PMIX_INFO_FREE(cd->info, cd->ninfo); } if (NULL != cd->apps) { - PMIX_APP_CREATE(cd->apps, cd->napps); + PMIX_APP_FREE(cd->apps, cd->napps); } if (NULL != cd->spcbfunc) { cd->spcbfunc(status, nspace, cd->cbdata); @@ -1134,7 +1156,6 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, "recvd SPAWN from %s:%d", peer->info->pname.nspace, peer->info->pname.rank); if (NULL == pmix_host_server.spawn) { - PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); return PMIX_ERR_NOT_SUPPORTED; } @@ -1200,7 +1221,8 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, } } } - /* we will construct any required iof request tracker upon completion of the spawn */ + /* we will construct any required iof request tracker upon completion of the spawn + * as we need the nspace of the spawned application! */ } /* add the directive to the end */ if (PMIX_PROC_IS_TOOL(peer)) { @@ -1242,7 +1264,7 @@ pmix_status_t pmix_server_spawn(pmix_peer_t *peer, } } /* call the local server */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; rc = pmix_host_server.spawn(&proc, cd->info, cd->ninfo, cd->apps, cd->napps, spcbfunc, cd); @@ -1326,15 +1348,11 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, } /* find/create the local tracker for this operation */ - if (NULL == (trk = get_tracker(procs, nprocs, PMIX_DISCONNECTNB_CMD))) { + if (NULL == (trk = get_tracker(NULL, procs, nprocs, PMIX_DISCONNECTNB_CMD))) { /* we don't have this tracker yet, so get a new one */ - if (NULL == (trk = new_tracker(procs, nprocs, PMIX_DISCONNECTNB_CMD))) { + if (NULL == (trk = new_tracker(NULL, procs, nprocs, PMIX_DISCONNECTNB_CMD))) { /* only if a bozo error occurs */ PMIX_ERROR_LOG(PMIX_ERROR); - /* DO NOT HANG */ - if (NULL != cbfunc) { - cbfunc(PMIX_ERROR, cd); - } rc = PMIX_ERROR; goto cleanup; } @@ -1352,7 +1370,6 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, /* add this contributor to the tracker so they get * notified when we are done */ - PMIX_RETAIN(cd); // prevent the caddy from being released when we return pmix_list_append(&trk->local_cbs, &cd->super); /* if all local contributions have been received, * let the local host's server know that we are at the @@ -1361,6 +1378,11 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, if (trk->def_complete && pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { rc = pmix_host_server.disconnect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); + if (PMIX_SUCCESS != rc) { + /* remove this contributor from the list - they will be notified + * by the switchyard */ + pmix_list_remove_item(&trk->local_cbs, &cd->super); + } } else { rc = PMIX_SUCCESS; } @@ -1470,9 +1492,9 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, } /* find/create the local tracker for this operation */ - if (NULL == (trk = get_tracker(procs, nprocs, PMIX_CONNECTNB_CMD))) { + if (NULL == (trk = get_tracker(NULL, procs, nprocs, PMIX_CONNECTNB_CMD))) { /* we don't have this tracker yet, so get a new one */ - if (NULL == (trk = new_tracker(procs, nprocs, PMIX_CONNECTNB_CMD))) { + if (NULL == (trk = new_tracker(NULL, procs, nprocs, PMIX_CONNECTNB_CMD))) { /* only if a bozo error occurs */ PMIX_ERROR_LOG(PMIX_ERROR); /* DO NOT HANG */ @@ -1496,17 +1518,7 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, /* add this contributor to the tracker so they get * notified when we are done */ - PMIX_RETAIN(cd); // prevent the caddy from being released when we return pmix_list_append(&trk->local_cbs, &cd->super); - /* if a timeout was specified, set it */ - if (0 < tv.tv_sec) { - PMIX_RETAIN(trk); - cd->trk = trk; - pmix_event_evtimer_set(pmix_globals.evbase, &cd->ev, - connect_timeout, cd); - pmix_event_evtimer_add(&cd->ev, &tv); - cd->event_active = true; - } /* if all local contributions have been received, * let the local host's server know that we are at the @@ -1515,9 +1527,23 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, if (trk->def_complete && pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { rc = pmix_host_server.connect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); + if (PMIX_SUCCESS != rc) { + /* remove this contributor from the list - they will be notified + * by the switchyard */ + pmix_list_remove_item(&trk->local_cbs, &cd->super); + } } else { rc = PMIX_SUCCESS; } + /* if a timeout was specified, set it */ + if (PMIX_SUCCESS == rc && 0 < tv.tv_sec) { + PMIX_RETAIN(trk); + cd->trk = trk; + pmix_event_evtimer_set(pmix_globals.evbase, &cd->ev, + connect_timeout, cd); + pmix_event_evtimer_add(&cd->ev, &tv); + cd->event_active = true; + } cleanup: if (NULL != procs) { @@ -1535,12 +1561,12 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, void *cbdata) { int32_t cnt; - pmix_status_t rc; + pmix_status_t rc, ret = PMIX_SUCCESS; pmix_status_t *codes = NULL; pmix_info_t *info = NULL; size_t ninfo=0, ncodes, n, k; pmix_regevents_info_t *reginfo; - pmix_peer_events_info_t *prev; + pmix_peer_events_info_t *prev = NULL; pmix_notify_caddy_t *cd; pmix_setup_caddy_t *scd; int i; @@ -1550,6 +1576,8 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, pmix_cmd_t cmd = PMIX_NOTIFY_CMD; pmix_proc_t *affected = NULL; size_t naffected = 0; + pmix_range_trkr_t rngtrk; + pmix_proc_t proc; pmix_output_verbose(2, pmix_server_globals.event_output, "recvd register events for peer %s:%d", @@ -1601,9 +1629,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, /* check the directives */ for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_EVENT_ENVIRO_LEVEL, PMIX_MAX_KEYLEN)) { - enviro_events = PMIX_INFO_TRUE(&info[n]); - } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_AFFECTED_PROC)) { if (NULL != affected) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); rc = PMIX_ERR_BAD_PARAM; @@ -1612,7 +1638,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, naffected = 1; PMIX_PROC_CREATE(affected, naffected); memcpy(affected, info[n].value.data.proc, sizeof(pmix_proc_t)); - } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROCS, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_EVENT_AFFECTED_PROCS)) { if (NULL != affected) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); rc = PMIX_ERR_BAD_PARAM; @@ -1624,6 +1650,14 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } } + /* check the codes for system events */ + for (n=0; n < ncodes; n++) { + if (PMIX_SYSTEM_EVENT(codes[n])) { + enviro_events = true; + break; + } + } + /* if they asked for enviro events, and our host doesn't support * register_events, then we cannot meet the request */ if (enviro_events && NULL == pmix_host_server.register_events) { @@ -1632,10 +1666,36 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, goto cleanup; } + /* if they didn't send us any codes, then they are registering a + * default event handler. In that case, check only for default + * handlers and add this request to it, if not already present */ + if (0 == ncodes) { + PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { + if (PMIX_MAX_ERR_CONSTANT == reginfo->code) { + /* both are default handlers */ + prev = PMIX_NEW(pmix_peer_events_info_t); + if (NULL == prev) { + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + PMIX_RETAIN(peer); + prev->peer = peer; + if (NULL != affected) { + PMIX_PROC_CREATE(prev->affected, naffected); + prev->naffected = naffected; + memcpy(prev->affected, affected, naffected * sizeof(pmix_proc_t)); + } + pmix_list_append(®info->peers, &prev->super); + break; + } + } + rc = PMIX_OPERATION_SUCCEEDED; + goto cleanup; + } + /* store the event registration info so we can call the registered * client when the server notifies the event */ - k=0; - do { + for (n=0; n < ncodes; n++) { found = false; PMIX_LIST_FOREACH(reginfo, &pmix_server_globals.events, pmix_regevents_info_t) { if (NULL == codes) { @@ -1649,35 +1709,28 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } else { if (PMIX_MAX_ERR_CONSTANT == reginfo->code) { continue; - } else if (codes[k] == reginfo->code) { + } else if (codes[n] == reginfo->code) { found = true; break; } } } if (found) { - /* found it - add this peer if we don't already have it */ - found = false; - PMIX_LIST_FOREACH(prev, ®info->peers, pmix_peer_events_info_t) { - if (prev->peer == peer) { - /* already have it */ - rc = PMIX_SUCCESS; - found = true; - break; - } + /* found it - add this request */ + prev = PMIX_NEW(pmix_peer_events_info_t); + if (NULL == prev) { + rc = PMIX_ERR_NOMEM; + goto cleanup; } - if (!found) { - /* get here if we don't already have this peer */ - prev = PMIX_NEW(pmix_peer_events_info_t); - if (NULL == prev) { - rc = PMIX_ERR_NOMEM; - goto cleanup; - } - PMIX_RETAIN(peer); - prev->peer = peer; - prev->enviro_events = enviro_events; - pmix_list_append(®info->peers, &prev->super); + PMIX_RETAIN(peer); + prev->peer = peer; + if (NULL != affected) { + PMIX_PROC_CREATE(prev->affected, naffected); + prev->naffected = naffected; + memcpy(prev->affected, affected, naffected * sizeof(pmix_proc_t)); } + prev->enviro_events = enviro_events; + pmix_list_append(®info->peers, &prev->super); } else { /* if we get here, then we didn't find an existing registration for this code */ reginfo = PMIX_NEW(pmix_regevents_info_t); @@ -1688,7 +1741,7 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, if (NULL == codes) { reginfo->code = PMIX_MAX_ERR_CONSTANT; } else { - reginfo->code = codes[k]; + reginfo->code = codes[n]; } pmix_list_append(&pmix_server_globals.events, ®info->super); prev = PMIX_NEW(pmix_peer_events_info_t); @@ -1698,14 +1751,23 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, } PMIX_RETAIN(peer); prev->peer = peer; + if (NULL != affected) { + PMIX_PROC_CREATE(prev->affected, naffected); + prev->naffected = naffected; + memcpy(prev->affected, affected, naffected * sizeof(pmix_proc_t)); + } prev->enviro_events = enviro_events; pmix_list_append(®info->peers, &prev->super); } - ++k; - } while (k < ncodes); + } /* if they asked for enviro events, call the local server */ if (enviro_events) { + /* if they don't support this, then we cannot do it */ + if (NULL == pmix_host_server.register_events) { + rc = PMIX_ERR_NOT_SUPPORTED; + goto cleanup; + } /* need to ensure the arrays don't go away until after the * host RM is done with them */ scd = PMIX_NEW(pmix_setup_caddy_t); @@ -1751,22 +1813,18 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, PMIX_INFO_FREE(scd->info, scd->ninfo); } PMIX_RELEASE(scd); - } else { - goto check; } + } else { + rc = PMIX_OPERATION_SUCCEEDED; } cleanup: pmix_output_verbose(2, pmix_server_globals.event_output, "server register events: ninfo =%lu rc =%d", ninfo, rc); - /* be sure to execute the callback */ - if (NULL != cbfunc) { - cbfunc(rc, cbdata); - } if (NULL != info) { PMIX_INFO_FREE(info, ninfo); } - if (PMIX_SUCCESS != rc) { + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { if (NULL != codes) { free(codes); } @@ -1776,11 +1834,13 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, return rc; } - check: /* check if any matching notifications have been cached */ - for (i=0; i < pmix_globals.notifications.size; i++) { - if (NULL == (cd = (pmix_notify_caddy_t*)pmix_ring_buffer_poke(&pmix_globals.notifications, i))) { - break; + rngtrk.procs = NULL; + rngtrk.nprocs = 0; + for (i=0; i < pmix_globals.max_events; i++) { + pmix_hotel_knock(&pmix_globals.notifications, i, (void**)&cd); + if (NULL == cd) { + continue; } found = false; if (NULL == codes) { @@ -1799,23 +1859,46 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, if (!found) { continue; } - /* if we were given specific targets, check if this is one */ + /* check if the affected procs (if given) match those they + * wanted to know about */ + if (!pmix_notify_check_affected(cd->affected, cd->naffected, + affected, naffected)) { + continue; + } + /* check the range */ + if (NULL == cd->targets) { + rngtrk.procs = &cd->source; + rngtrk.nprocs = 1; + } else { + rngtrk.procs = cd->targets; + rngtrk.nprocs = cd->ntargets; + } + rngtrk.range = cd->range; + PMIX_LOAD_PROCID(&proc, peer->info->pname.nspace, peer->info->pname.rank); + if (!pmix_notify_check_range(&rngtrk, &proc)) { + continue; + } + /* if we were given specific targets, check if this is one */ + found = false; if (NULL != cd->targets) { matched = false; for (n=0; n < cd->ntargets; n++) { - if (0 != strncmp(peer->info->pname.nspace, cd->targets[n].nspace, PMIX_MAX_NSLEN)) { - continue; - } /* if the source of the event is the same peer just registered, then ignore it * as the event notification system will have already locally * processed it */ - if (0 == strncmp(peer->info->pname.nspace, cd->source.nspace, PMIX_MAX_NSLEN) && - peer->info->pname.rank == cd->source.rank) { + if (PMIX_CHECK_PROCID(&cd->source, &peer->info->pname)) { continue; } - if (PMIX_RANK_WILDCARD == cd->targets[n].rank || - peer->info->pname.rank == cd->targets[n].rank) { + if (PMIX_CHECK_PROCID(&peer->info->pname, &cd->targets[n])) { matched = true; + /* track the number of targets we have left to notify */ + --cd->nleft; + /* if this is the last one, then evict this event + * from the cache */ + if (0 == cd->nleft) { + pmix_hotel_checkout(&pmix_globals.notifications, cd->room); + found = true; // mark that we should release cd + } break; } } @@ -1824,58 +1907,61 @@ pmix_status_t pmix_server_register_events(pmix_peer_t *peer, continue; } } - /* if they specified affected proc(s) they wanted to know about, check */ - if (!pmix_notify_check_affected(cd->affected, cd->naffected, - affected, naffected)) { - continue; - } + /* all matches - notify */ relay = PMIX_NEW(pmix_buffer_t); if (NULL == relay) { /* nothing we can do */ PMIX_ERROR_LOG(PMIX_ERR_NOMEM); - rc = PMIX_ERR_NOMEM; + ret = PMIX_ERR_NOMEM; break; } /* pack the info data stored in the event */ - PMIX_BFROPS_PACK(rc, peer, relay, &cmd, 1, PMIX_COMMAND); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); + PMIX_BFROPS_PACK(ret, peer, relay, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); break; } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->status, 1, PMIX_STATUS); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); + PMIX_BFROPS_PACK(ret, peer, relay, &cd->status, 1, PMIX_STATUS); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); break; } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->source, 1, PMIX_PROC); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); + PMIX_BFROPS_PACK(ret, peer, relay, &cd->source, 1, PMIX_PROC); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); break; } - PMIX_BFROPS_PACK(rc, peer, relay, &cd->ninfo, 1, PMIX_SIZE); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); + PMIX_BFROPS_PACK(ret, peer, relay, &cd->ninfo, 1, PMIX_SIZE); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); break; } if (0 < cd->ninfo) { - PMIX_BFROPS_PACK(rc, peer, relay, cd->info, cd->ninfo, PMIX_INFO); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); + PMIX_BFROPS_PACK(ret, peer, relay, cd->info, cd->ninfo, PMIX_INFO); + if (PMIX_SUCCESS != ret) { + PMIX_ERROR_LOG(ret); break; } } - PMIX_SERVER_QUEUE_REPLY(peer, 0, relay); - } - if (!enviro_events) { - if (NULL != codes) { - free(codes); + PMIX_SERVER_QUEUE_REPLY(ret, peer, 0, relay); + if (PMIX_SUCCESS != ret) { + PMIX_RELEASE(relay); } + if (found) { + PMIX_RELEASE(cd); + } + } + + if (NULL != codes) { + free(codes); } if (NULL != affected) { PMIX_PROC_FREE(affected, naffected); } - + if (PMIX_SUCCESS != ret) { + rc = ret; + } return rc; } @@ -1956,6 +2042,13 @@ static void intermed_step(pmix_status_t status, void *cbdata) goto complete; } + /* since our host is going to send this everywhere, it may well + * come back to us. We already processed it, so mark it here + * to ensure we don't do it again. We previously inserted the + * PMIX_SERVER_INTERNAL_NOTIFY key at the very end of the + * info array - just overwrite that position */ + PMIX_INFO_LOAD(&cd->info[cd->ninfo-1], PMIX_EVENT_PROXY, &pmix_globals.myid, PMIX_PROC); + /* pass it to our host RM for distribution */ rc = pmix_host_server.notify_event(cd->status, &cd->source, cd->range, cd->info, cd->ninfo, local_cbfunc, cd); @@ -1974,6 +2067,11 @@ static void intermed_step(pmix_status_t status, void *cbdata) PMIX_RELEASE(cd); } +/* Receive an event sent by the client library. Since it was sent + * to us by one client, we have to both process it locally to ensure + * we notify all relevant local clients AND (assuming a range other + * than LOCAL) deliver to our host, requesting that they send it + * to all peer servers in the current session */ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, pmix_buffer_t *buf, pmix_op_cbfunc_t cbfunc, @@ -1982,11 +2080,12 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, int32_t cnt; pmix_status_t rc; pmix_notify_caddy_t *cd; - size_t ninfo; + size_t ninfo, n; pmix_output_verbose(2, pmix_server_globals.event_output, - "%s:%d recvd event notification from client", - pmix_globals.myid.nspace, pmix_globals.myid.rank); + "%s:%d recvd event notification from client %s:%d", + pmix_globals.myid.nspace, pmix_globals.myid.rank, + peer->info->pname.nspace, peer->info->pname.rank); cd = PMIX_NEW(pmix_notify_caddy_t); if (NULL == cd) { @@ -1995,8 +2094,7 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, cd->cbfunc = cbfunc; cd->cbdata = cbdata; /* set the source */ - (void)strncpy(cd->source.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - cd->source.rank = peer->info->pname.rank; + PMIX_LOAD_PROCID(&cd->source, peer->info->pname.nspace, peer->info->pname.rank); /* unpack status */ cnt = 1; @@ -2035,6 +2133,18 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, goto exit; } } + + /* check to see if we already processed this event - it is possible + * that a local client "echoed" it back to us and we want to avoid + * a potential infinite loop */ + for (n=0; n < ninfo; n++) { + if (PMIX_CHECK_KEY(&cd->info[n], PMIX_SERVER_INTERNAL_NOTIFY)) { + /* yep, we did - so don't do it again! */ + rc = PMIX_OPERATION_SUCCEEDED; + goto exit; + } + } + /* add an info object to mark that we recvd this internally */ PMIX_INFO_LOAD(&cd->info[ninfo], PMIX_SERVER_INTERNAL_NOTIFY, NULL, PMIX_BOOL); /* process it */ @@ -2045,12 +2155,13 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, intermed_step, cd))) { goto exit; } - /* tell the switchyard we will handle it from here */ - return PMIX_SUCCESS; + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(cd); + } + return rc; exit: PMIX_RELEASE(cd); - cbfunc(rc, cbdata); return rc; } @@ -2063,14 +2174,14 @@ pmix_status_t pmix_server_query(pmix_peer_t *peer, pmix_status_t rc; pmix_query_caddy_t *cd; pmix_proc_t proc; + pmix_cb_t cb; + size_t n, p; + pmix_list_t results; + pmix_kval_t *kv, *kvnxt; pmix_output_verbose(2, pmix_server_globals.base_output, "recvd query from client"); - if (NULL == pmix_host_server.query) { - return PMIX_ERR_NOT_SUPPORTED; - } - cd = PMIX_NEW(pmix_query_caddy_t); if (NULL == cd) { return PMIX_ERR_NOMEM; @@ -2081,36 +2192,136 @@ pmix_status_t pmix_server_query(pmix_peer_t *peer, PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->nqueries, &cnt, PMIX_SIZE); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto exit; + PMIX_RELEASE(cd); + return rc; } /* unpack the queries */ if (0 < cd->nqueries) { PMIX_QUERY_CREATE(cd->queries, cd->nqueries); if (NULL == cd->queries) { rc = PMIX_ERR_NOMEM; - goto exit; + PMIX_RELEASE(cd); + return rc; } cnt = cd->nqueries; PMIX_BFROPS_UNPACK(rc, peer, buf, cd->queries, &cnt, PMIX_QUERY); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); - goto exit; + PMIX_RELEASE(cd); + return rc; } } + /* check the directives to see if they want us to refresh + * the local cached results - if we wanted to optimize this + * more, we would check each query and allow those that don't + * want to be refreshed to be executed locally, and those that + * did would be sent to the host. However, for now we simply + * determine that if we don't have it, then ask for everything */ + memset(proc.nspace, 0, PMIX_MAX_NSLEN+1); + proc.rank = PMIX_RANK_INVALID; + PMIX_CONSTRUCT(&results, pmix_list_t); + + for (n=0; n < cd->nqueries; n++) { + for (p=0; p < cd->queries[n].nqual; p++) { + if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_QUERY_REFRESH_CACHE)) { + if (PMIX_INFO_TRUE(&cd->queries[n].qualifiers[p])) { + PMIX_LIST_DESTRUCT(&results); + goto query; + } + } else if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_PROCID)) { + PMIX_LOAD_NSPACE(proc.nspace, cd->queries[n].qualifiers[p].value.data.proc->nspace); + proc.rank = cd->queries[n].qualifiers[p].value.data.proc->rank; + } else if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_NSPACE)) { + PMIX_LOAD_NSPACE(proc.nspace, cd->queries[n].qualifiers[p].value.data.string); + } else if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_RANK)) { + proc.rank = cd->queries[n].qualifiers[p].value.data.rank; + } else if (PMIX_CHECK_KEY(&cd->queries[n].qualifiers[p], PMIX_HOSTNAME)) { + if (0 != strcmp(cd->queries[n].qualifiers[p].value.data.string, pmix_globals.hostname)) { + /* asking about a different host, so ask for the info */ + PMIX_LIST_DESTRUCT(&results); + goto query; + } + } + } + /* we get here if a refresh isn't required - first try a local + * "get" on the data to see if we already have it */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + cb.copy = false; + /* set the proc */ + if (PMIX_RANK_INVALID == proc.rank && + 0 == strlen(proc.nspace)) { + /* use our id */ + cb.proc = &pmix_globals.myid; + } else { + if (0 == strlen(proc.nspace)) { + /* use our nspace */ + PMIX_LOAD_NSPACE(cb.proc->nspace, pmix_globals.myid.nspace); + } + if (PMIX_RANK_INVALID == proc.rank) { + /* user the wildcard rank */ + proc.rank = PMIX_RANK_WILDCARD; + } + cb.proc = &proc; + } + for (p=0; NULL != cd->queries[n].keys[p]; p++) { + cb.key = cd->queries[n].keys[p]; + PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); + if (PMIX_SUCCESS != rc) { + /* needs to be passed to the host */ + PMIX_LIST_DESTRUCT(&results); + PMIX_DESTRUCT(&cb); + goto query; + } + /* need to retain this result */ + PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &cb.kvs, pmix_kval_t) { + pmix_list_remove_item(&cb.kvs, &kv->super); + pmix_list_append(&results, &kv->super); + } + PMIX_DESTRUCT(&cb); + } + } + + /* if we get here, then all queries were completely locally + * resolved, so construct the results for return */ + rc = PMIX_ERR_NOT_FOUND; + if (0 < (cd->ninfo = pmix_list_get_size(&results))) { + PMIX_INFO_CREATE(cd->info, cd->ninfo); + n = 0; + PMIX_LIST_FOREACH_SAFE(kv, kvnxt, &results, pmix_kval_t) { + PMIX_LOAD_KEY(cd->info[n].key, kv->key); + rc = pmix_value_xfer(&cd->info[n].value, kv->value); + if (PMIX_SUCCESS != rc) { + PMIX_INFO_FREE(cd->info, cd->ninfo); + cd->info = NULL; + cd->ninfo = 0; + break; + } + ++n; + } + } + /* done with the list of results */ + PMIX_LIST_DESTRUCT(&results); + /* we can just call the cbfunc here as we are already + * in an event - let our internal cbfunc do a threadshift + * if necessary */ + cbfunc(PMIX_SUCCESS, cd->info, cd->ninfo, cd, NULL, NULL); + return PMIX_SUCCESS; + + query: + if (NULL == pmix_host_server.query) { + PMIX_RELEASE(cd); + return PMIX_ERR_NOT_SUPPORTED; + } + /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); - proc.rank = peer->info->pname.rank; + PMIX_LOAD_PROCID(&proc, peer->info->pname.nspace, peer->info->pname.rank); /* ask the host for the info */ if (PMIX_SUCCESS != (rc = pmix_host_server.query(&proc, cd->queries, cd->nqueries, cbfunc, cd))) { - goto exit; + PMIX_RELEASE(cd); } - return PMIX_SUCCESS; - - exit: - PMIX_RELEASE(cd); return rc; } @@ -2142,7 +2353,7 @@ pmix_status_t pmix_server_log(pmix_peer_t *peer, * the request itself */ /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; cd = PMIX_NEW(pmix_shift_caddy_t); @@ -2270,7 +2481,7 @@ pmix_status_t pmix_server_alloc(pmix_peer_t *peer, } /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; /* ask the host to execute the request */ @@ -2302,7 +2513,7 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, int32_t cnt, m; pmix_status_t rc; pmix_query_caddy_t *cd; - pmix_nspace_t *nptr, *tmp; + pmix_namespace_t *nptr, *tmp; pmix_peer_t *pr; pmix_proc_t proc; size_t n; @@ -2325,6 +2536,8 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, } cd->cbdata = cbdata; + PMIX_CONSTRUCT(&epicache, pmix_list_t); + /* unpack the number of targets */ cnt = 1; PMIX_BFROPS_UNPACK(rc, peer, buf, &cd->ntargets, &cnt, PMIX_SIZE); @@ -2343,7 +2556,6 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, } /* check targets to find proper place to put any epilog requests */ - PMIX_CONSTRUCT(&epicache, pmix_list_t); if (NULL == cd->targets) { epicd = PMIX_NEW(pmix_srvr_epi_caddy_t); epicd->epi = &peer->nptr->epilog; @@ -2352,14 +2564,14 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, for (n=0; n < cd->ntargets; n++) { /* find the nspace of this proc */ nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_nspace_t) { + PMIX_LIST_FOREACH(tmp, &pmix_server_globals.nspaces, pmix_namespace_t) { if (0 == strcmp(tmp->nspace, cd->targets[n].nspace)) { nptr = tmp; break; } } if (NULL == nptr) { - nptr = PMIX_NEW(pmix_nspace_t); + nptr = PMIX_NEW(pmix_namespace_t); if (NULL == nptr) { rc = PMIX_ERR_NOMEM; goto exit; @@ -2523,7 +2735,6 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, rc = PMIX_ERR_CONFLICTING_CLEANUP_DIRECTIVES; PMIX_LIST_DESTRUCT(&cachedirs); PMIX_LIST_DESTRUCT(&cachefiles); - PMIX_LIST_DESTRUCT(&epicache); goto exit; } } @@ -2568,13 +2779,13 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, PMIX_LIST_DESTRUCT(&cachefiles); if (cnt == (int)cd->ninfo) { /* nothing more to do */ - rc = PMIX_SUCCESS; + rc = PMIX_OPERATION_SUCCEEDED; goto exit; } } /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; /* ask the host to execute the request */ @@ -2584,10 +2795,12 @@ pmix_status_t pmix_server_job_ctrl(pmix_peer_t *peer, cbfunc, cd))) { goto exit; } + PMIX_LIST_DESTRUCT(&epicache); return PMIX_SUCCESS; exit: PMIX_RELEASE(cd); + PMIX_LIST_DESTRUCT(&epicache); return rc; } @@ -2666,7 +2879,7 @@ pmix_status_t pmix_server_monitor(pmix_peer_t *peer, } /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; /* ask the host to execute the request */ @@ -2725,7 +2938,7 @@ pmix_status_t pmix_server_get_credential(pmix_peer_t *peer, } /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; /* ask the host to execute the request */ @@ -2790,7 +3003,7 @@ pmix_status_t pmix_server_validate_credential(pmix_peer_t *peer, } /* setup the requesting peer name */ - (void)strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc.nspace, peer->info->pname.nspace, PMIX_MAX_NSLEN); proc.rank = peer->info->pname.rank; /* ask the host to execute the request */ @@ -2817,9 +3030,8 @@ pmix_status_t pmix_server_iofreg(pmix_peer_t *peer, pmix_iof_req_t *req; bool notify, match; size_t n; - int i; - pmix_setup_caddy_t *occupant; pmix_buffer_t *msg; + pmix_iof_cache_t *iof, *ionext; pmix_output_verbose(2, pmix_server_globals.iof_output, "recvd IOF PULL request from client"); @@ -2916,54 +3128,60 @@ pmix_status_t pmix_server_iofreg(pmix_peer_t *peer, pmix_list_append(&pmix_globals.iof_requests, &req->super); } /* process any cached IO */ - for (i=0; i < PMIX_IOF_HOTEL_SIZE; i++) { - pmix_hotel_knock(&pmix_server_globals.iof, PMIX_IOF_HOTEL_SIZE-i-1, (void**)&occupant); - if (NULL != occupant) { - if (!(occupant->channels & req->channels)) { - continue; - } - /* if the source matches the request, then forward this along */ - if (0 != strncmp(occupant->procs->nspace, req->pname.nspace, PMIX_MAX_NSLEN) || - (PMIX_RANK_WILDCARD != req->pname.rank && occupant->procs->rank != req->pname.rank)) { - continue; - } - /* setup the msg */ - if (NULL == (msg = PMIX_NEW(pmix_buffer_t))) { - PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - rc = PMIX_ERR_OUT_OF_RESOURCE; - break; - } - /* provide the source */ - PMIX_BFROPS_PACK(rc, req->peer, msg, occupant->procs, 1, PMIX_PROC); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - break; - } - /* provide the channel */ - PMIX_BFROPS_PACK(rc, req->peer, msg, &occupant->channels, 1, PMIX_IOF_CHANNEL); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - break; - } - /* pack the data */ - PMIX_BFROPS_PACK(rc, req->peer, msg, occupant->bo, 1, PMIX_BYTE_OBJECT); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - break; - } - /* send it to the requestor */ - PMIX_PTL_SEND_ONEWAY(rc, req->peer, msg, PMIX_PTL_TAG_IOF); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(msg); - } - /* remove it from the hotel since it has now been forwarded */ - pmix_hotel_checkout(&pmix_server_globals.iof, PMIX_IOF_HOTEL_SIZE-i-1); - PMIX_RELEASE(occupant); + PMIX_LIST_FOREACH_SAFE(iof, ionext, &pmix_server_globals.iof, pmix_iof_cache_t) { + /* if the channels don't match, then ignore it */ + if (!(iof->channel & req->channels)) { + continue; } + /* if the source does not match the request, then ignore it */ + if (!PMIX_CHECK_PROCID(&iof->source, &req->pname)) { + continue; + } + /* never forward back to the source! This can happen if the source + * is a launcher */ + if (PMIX_CHECK_PROCID(&iof->source, &req->peer->info->pname)) { + continue; + } + pmix_output_verbose(2, pmix_server_globals.iof_output, + "PMIX:SERVER:IOFREQ delivering cached IOF from %s:%d to %s:%d", + iof->source.nspace, iof->source.rank, + req->peer->info->pname.nspace, req->peer->info->pname.rank); + /* setup the msg */ + if (NULL == (msg = PMIX_NEW(pmix_buffer_t))) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + rc = PMIX_ERR_OUT_OF_RESOURCE; + break; + } + /* provide the source */ + PMIX_BFROPS_PACK(rc, req->peer, msg, &iof->source, 1, PMIX_PROC); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + break; + } + /* provide the channel */ + PMIX_BFROPS_PACK(rc, req->peer, msg, &iof->channel, 1, PMIX_IOF_CHANNEL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + break; + } + /* pack the data */ + PMIX_BFROPS_PACK(rc, req->peer, msg, iof->bo, 1, PMIX_BYTE_OBJECT); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + break; + } + /* send it to the requestor */ + PMIX_PTL_SEND_ONEWAY(rc, req->peer, msg, PMIX_PTL_TAG_IOF); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + } + /* remove it from the list since it has now been forwarded */ + pmix_list_remove_item(&pmix_server_globals.iof, &iof->super); + PMIX_RELEASE(iof); } } if (notify) { @@ -3082,7 +3300,7 @@ pmix_status_t pmix_server_iofstdin(pmix_peer_t *peer, } /* pass the data to the host */ - (void)strncpy(source.nspace, peer->nptr->nspace, PMIX_MAX_NSLEN); + pmix_strncpy(source.nspace, peer->nptr->nspace, PMIX_MAX_NSLEN); source.rank = peer->info->pname.rank; if (PMIX_SUCCESS != (rc = pmix_host_server.push_stdin(&source, cd->procs, cd->nprocs, cd->info, cd->ninfo, cd->bo, @@ -3099,6 +3317,9 @@ pmix_status_t pmix_server_iofstdin(pmix_peer_t *peer, /***** INSTANCE SERVER LIBRARY CLASSES *****/ static void tcon(pmix_server_trkr_t *t) { + t->event_active = false; + t->lost_connection = false; + t->id = NULL; memset(t->pname.nspace, 0, PMIX_MAX_NSLEN+1); t->pname.rank = PMIX_RANK_UNDEF; t->pcs = NULL; @@ -3118,6 +3339,9 @@ static void tcon(pmix_server_trkr_t *t) } static void tdes(pmix_server_trkr_t *t) { + if (NULL != t->id) { + free(t->id); + } PMIX_DESTRUCT_LOCK(&t->lock); if (NULL != t->pcs) { free(t->pcs); @@ -3165,6 +3389,8 @@ static void scadcon(pmix_setup_caddy_t *p) p->ncodes = 0; p->procs = NULL; p->nprocs = 0; + p->apps = NULL; + p->napps = 0; p->server_object = NULL; p->nlocalprocs = 0; p->info = NULL; @@ -3172,6 +3398,7 @@ static void scadcon(pmix_setup_caddy_t *p) p->keys = NULL; p->channels = PMIX_FWD_NO_CHANNELS; p->bo = NULL; + p->nbo = 0; p->cbfunc = NULL; p->opcbfunc = NULL; p->setupcbfunc = NULL; @@ -3184,6 +3411,13 @@ static void scaddes(pmix_setup_caddy_t *p) if (NULL != p->peer) { PMIX_RELEASE(p->peer); } + PMIX_PROC_FREE(p->procs, p->nprocs); + if (NULL != p->apps) { + PMIX_APP_FREE(p->apps, p->napps); + } + if (NULL != p->bo) { + PMIX_BYTE_OBJECT_FREE(p->bo, p->nbo); + } PMIX_DESTRUCT_LOCK(&p->lock); } PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_setup_caddy_t, @@ -3192,12 +3426,20 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_setup_caddy_t, static void ncon(pmix_notify_caddy_t *p) { + struct timespec tp; + PMIX_CONSTRUCT_LOCK(&p->lock); + clock_gettime(CLOCK_MONOTONIC, &tp); + p->ts = tp.tv_sec; + p->room = -1; memset(p->source.nspace, 0, PMIX_MAX_NSLEN+1); p->source.rank = PMIX_RANK_UNDEF; p->range = PMIX_RANGE_UNDEF; p->targets = NULL; p->ntargets = 0; + p->nleft = SIZE_MAX; + p->affected = NULL; + p->naffected = 0; p->nondefault = false; p->info = NULL; p->ninfo = 0; @@ -3208,6 +3450,7 @@ static void ndes(pmix_notify_caddy_t *p) if (NULL != p->info) { PMIX_INFO_FREE(p->info, p->ninfo); } + PMIX_PROC_FREE(p->affected, p->naffected); if (NULL != p->targets) { free(p->targets); } @@ -3275,12 +3518,17 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_local_t, static void prevcon(pmix_peer_events_info_t *p) { p->peer = NULL; + p->affected = NULL; + p->naffected = 0; } static void prevdes(pmix_peer_events_info_t *p) { if (NULL != p->peer) { PMIX_RELEASE(p->peer); } + if (NULL != p->affected) { + PMIX_PROC_FREE(p->affected, p->naffected); + } } PMIX_CLASS_INSTANCE(pmix_peer_events_info_t, pmix_list_item_t, @@ -3321,3 +3569,15 @@ static void ildes(pmix_inventory_rollup_t *p) PMIX_CLASS_INSTANCE(pmix_inventory_rollup_t, pmix_object_t, ilcon, ildes); + +static void iocon(pmix_iof_cache_t *p) +{ + p->bo = NULL; +} +static void iodes(pmix_iof_cache_t *p) +{ + PMIX_BYTE_OBJECT_FREE(p->bo, 1); // macro protects against NULL +} +PMIX_CLASS_INSTANCE(pmix_iof_cache_t, + pmix_list_item_t, + iocon, iodes); diff --git a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h index e90137c90c..06fddc1fe9 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h +++ b/opal/mca/pmix/pmix3x/pmix/src/server/pmix_server_ops.h @@ -1,19 +1,24 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. - * Copyright (c) 2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2016-2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ */ #ifndef PMIX_SERVER_OPS_H #define PMIX_SERVER_OPS_H +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif + #include #include "src/include/types.h" #include @@ -57,6 +62,12 @@ typedef struct { size_t napps; pmix_iof_channel_t channels; pmix_byte_object_t *bo; + size_t nbo; + /* timestamp receipt of the notification so we + * can evict the oldest one if we get overwhelmed */ + time_t ts; + /* what room of the hotel they are in */ + int room; pmix_op_cbfunc_t opcbfunc; pmix_dmodex_response_fn_t cbfunc; pmix_setup_application_cbfunc_t setupcbfunc; @@ -120,6 +131,8 @@ typedef struct { pmix_list_item_t super; pmix_peer_t *peer; bool enviro_events; + pmix_proc_t *affected; + size_t naffected; } pmix_peer_events_info_t; PMIX_CLASS_DECLARATION(pmix_peer_events_info_t); @@ -130,6 +143,14 @@ typedef struct { } pmix_regevents_info_t; PMIX_CLASS_DECLARATION(pmix_regevents_info_t); +typedef struct { + pmix_list_item_t super; + pmix_proc_t source; + pmix_iof_channel_t channel; + pmix_byte_object_t *bo; +} pmix_iof_cache_t; +PMIX_CLASS_DECLARATION(pmix_iof_cache_t); + typedef struct { pmix_list_t nspaces; // list of pmix_nspace_t for the nspaces we know about pmix_pointer_array_t clients; // array of pmix_peer_t local clients @@ -138,9 +159,11 @@ typedef struct { pmix_list_t local_reqs; // list of pmix_dmdx_local_t awaiting arrival of data from local neighbours pmix_list_t gdata; // cache of data given to me for passing to all clients pmix_list_t events; // list of pmix_regevents_info_t registered events - pmix_hotel_t iof; // IO to be forwarded to clients + pmix_list_t iof; // IO to be forwarded to clients + size_t max_iof_cache; // max number of IOF messages to cache bool tool_connections_allowed; char *tmpdir; // temporary directory for this server + char *system_tmpdir; // system tmpdir // verbosity for server get operations int get_output; int get_verbose; @@ -194,8 +217,8 @@ typedef struct { bool pmix_server_trk_update(pmix_server_trkr_t *trk); -void pmix_pending_nspace_requests(pmix_nspace_t *nptr); -pmix_status_t pmix_pending_resolve(pmix_nspace_t *nptr, pmix_rank_t rank, +void pmix_pending_nspace_requests(pmix_namespace_t *nptr); +pmix_status_t pmix_pending_resolve(pmix_namespace_t *nptr, pmix_rank_t rank, pmix_status_t status, pmix_dmdx_local_t *lcd); @@ -312,7 +335,11 @@ void pmix_server_message_handler(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); +void pmix_server_purge_events(pmix_peer_t *peer, + pmix_proc_t *proc); + PMIX_EXPORT extern pmix_server_module_t pmix_host_server; PMIX_EXPORT extern pmix_server_globals_t pmix_server_globals; + #endif // PMIX_SERVER_OPS_H diff --git a/opal/mca/pmix/pmix3x/pmix/src/threads/mutex_unix.h b/opal/mca/pmix/pmix3x/pmix/src/threads/mutex_unix.h index f61d549923..229be8f177 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/threads/mutex_unix.h +++ b/opal/mca/pmix/pmix3x/pmix/src/threads/mutex_unix.h @@ -77,14 +77,14 @@ PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\ + .m_lock_atomic = {PMIX_ATOMIC_LOCK_UNLOCKED}, \ } #else #define PMIX_MUTEX_STATIC_INIT \ { \ .super = PMIX_OBJ_STATIC_INIT(pmix_mutex_t), \ .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\ + .m_lock_atomic = {PMIX_ATOMIC_LOCK_UNLOCKED}, \ } #endif @@ -98,14 +98,14 @@ PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\ + .m_lock_atomic = {PMIX_ATOMIC_LOCK_UNLOCKED}, \ } #else #define PMIX_RECURSIVE_MUTEX_STATIC_INIT \ { \ .super = PMIX_OBJ_STATIC_INIT(pmix_mutex_t), \ .m_lock_pthread = PMIX_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\ + .m_lock_atomic = {PMIX_ATOMIC_LOCK_UNLOCKED}, \ } #endif diff --git a/opal/mca/pmix/pmix3x/pmix/src/threads/thread_usage.h b/opal/mca/pmix/pmix3x/pmix/src/threads/thread_usage.h index 59825645c9..ff3e504180 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/threads/thread_usage.h +++ b/opal/mca/pmix/pmix3x/pmix/src/threads/thread_usage.h @@ -33,31 +33,30 @@ /** - * Use an atomic operation for increment/decrement if pmix_using_threads() - * indicates that threads are in use by the application or library. + * Use an atomic operation for increment/decrement */ #define PMIX_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix) \ -static inline type pmix_thread_ ## name ## _fetch_ ## suffix (volatile type *addr, type delta) \ +static inline type pmix_thread_ ## name ## _fetch_ ## suffix (pmix_atomic_ ## type *addr, type delta) \ { \ return pmix_atomic_ ## name ## _fetch_ ## suffix (addr, delta); \ } \ \ -static inline type pmix_thread_fetch_ ## name ## _ ## suffix (volatile type *addr, type delta) \ +static inline type pmix_thread_fetch_ ## name ## _ ## suffix (pmix_atomic_ ## type *addr, type delta) \ { \ return pmix_atomic_fetch_ ## name ## _ ## suffix (addr, delta); \ } #define PMIX_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \ -static inline bool pmix_thread_compare_exchange_strong_ ## suffix (volatile addr_type *addr, type *compare, type value) \ +static inline bool pmix_thread_compare_exchange_strong_ ## suffix (pmix_atomic_ ## addr_type *addr, type *compare, type value) \ { \ - return pmix_atomic_compare_exchange_strong_ ## suffix ((volatile type *) addr, compare, value); \ + return pmix_atomic_compare_exchange_strong_ ## suffix (addr, (addr_type *) compare, (addr_type) value); \ } #define PMIX_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \ -static inline type pmix_thread_swap_ ## suffix (volatile addr_type *ptr, type newvalue) \ +static inline type pmix_thread_swap_ ## suffix (pmix_atomic_ ## addr_type *ptr, type newvalue) \ { \ - return pmix_atomic_swap_ ## suffix ((volatile type *) ptr, newvalue); \ + return (type) pmix_atomic_swap_ ## suffix (ptr, (addr_type) newvalue); \ } PMIX_THREAD_DEFINE_ATOMIC_OP(int32_t, add, +, 32) @@ -112,13 +111,13 @@ PMIX_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) #define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_32 pmix_thread_compare_exchange_strong_32 #define PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_32 pmix_thread_compare_exchange_strong_32 -#define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) pmix_thread_compare_exchange_strong_ptr ((volatile intptr_t *) x, (void *) y, (void *) z) +#define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) pmix_thread_compare_exchange_strong_ptr ((pmix_atomic_intptr_t *) x, (intptr_t *) y, (intptr_t) z) #define PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR PMIX_THREAD_COMPARE_EXCHANGE_STRONG_PTR #define PMIX_THREAD_SWAP_32 pmix_thread_swap_32 #define PMIX_ATOMIC_SWAP_32 pmix_thread_swap_32 -#define PMIX_THREAD_SWAP_PTR(x, y) pmix_thread_swap_ptr ((volatile intptr_t *) x, (void *) y) +#define PMIX_THREAD_SWAP_PTR(x, y) pmix_thread_swap_ptr ((pmix_atomic_intptr_t *) x, (intptr_t) y) #define PMIX_ATOMIC_SWAP_PTR PMIX_THREAD_SWAP_PTR /* define 64-bit macros is 64-bit atomic math is available */ diff --git a/opal/mca/pmix/pmix3x/pmix/src/threads/wait_sync.h b/opal/mca/pmix/pmix3x/pmix/src/threads/wait_sync.h index 311ecbfe7f..225c8f157b 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/threads/wait_sync.h +++ b/opal/mca/pmix/pmix3x/pmix/src/threads/wait_sync.h @@ -28,7 +28,7 @@ BEGIN_C_DECLS typedef struct pmix_wait_sync_t { - int32_t count; + pmix_atomic_int32_t count; int32_t status; pthread_cond_t condition; pthread_mutex_t lock; diff --git a/opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c b/opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c index 28d35d301d..effa1190ff 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c +++ b/opal/mca/pmix/pmix3x/pmix/src/tool/pmix_tool.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2014 Artem Y. Polyakov . @@ -63,6 +63,7 @@ #include "src/runtime/pmix_rte.h" #include "src/mca/bfrops/base/base.h" #include "src/mca/gds/base/base.h" +#include "src/mca/pnet/base/base.h" #include "src/mca/ptl/base/base.h" #include "src/mca/psec/psec.h" #include "src/include/pmix_globals.h" @@ -195,7 +196,7 @@ static void tool_iof_handler(struct pmix_peer_t *pr, pmix_status_t rc; pmix_output_verbose(2, pmix_client_globals.iof_output, - "recvd IOF"); + "recvd IOF with %d bytes", (int)buf->bytes_used); /* if the buffer is empty, they are simply closing the channel */ if (0 == buf->bytes_used) { @@ -290,7 +291,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, * rank should be known. So return them here if * requested */ if (NULL != proc) { - (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); proc->rank = pmix_globals.myid.rank; } ++pmix_globals.init_cntr; @@ -327,10 +328,29 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, /* they want us to forward our stdin to someone */ fwd_stdin = true; } else if (0 == strncmp(info[n].key, PMIX_LAUNCHER, PMIX_MAX_KEYLEN)) { - ptype = PMIX_PROC_LAUNCHER; + ptype |= PMIX_PROC_LAUNCHER; + } else if (0 == strncmp(info[n].key, PMIX_SERVER_TMPDIR, PMIX_MAX_KEYLEN)) { + pmix_server_globals.tmpdir = strdup(info[n].value.data.string); + } else if (0 == strncmp(info[n].key, PMIX_SYSTEM_TMPDIR, PMIX_MAX_KEYLEN)) { + pmix_server_globals.system_tmpdir = strdup(info[n].value.data.string); } } } + if (NULL == pmix_server_globals.tmpdir) { + if (NULL == (evar = getenv("PMIX_SERVER_TMPDIR"))) { + pmix_server_globals.tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.tmpdir = strdup(evar); + } + } + if (NULL == pmix_server_globals.system_tmpdir) { + if (NULL == (evar = getenv("PMIX_SYSTEM_TMPDIR"))) { + pmix_server_globals.system_tmpdir = strdup(pmix_tmp_directory()); + } else { + pmix_server_globals.system_tmpdir = strdup(evar); + } + } + if ((nspace_given && !rank_given) || (!nspace_given && rank_given)) { /* can't have one and not the other */ @@ -387,7 +407,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, /* if we are a launcher, then we also need to act as a server, * so setup the server-related structures here */ - if (PMIX_PROC_LAUNCHER == ptype) { + if (PMIX_PROC_LAUNCHER_ACT & ptype) { if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { PMIX_ERROR_LOG(rc); if (NULL != nspace) { @@ -401,14 +421,6 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, } /* setup the function pointers */ memset(&pmix_host_server, 0, sizeof(pmix_server_module_t)); - /* setup our tmpdir */ - if (NULL == pmix_server_globals.tmpdir) { - if (NULL == (evar = getenv("PMIX_SERVER_TMPDIR"))) { - pmix_server_globals.tmpdir = strdup(pmix_tmp_directory()); - } else { - pmix_server_globals.tmpdir = strdup(evar); - } - } } /* setup the runtime - this init's the globals, @@ -427,7 +439,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, } /* if we were given a name, then set it now */ if (nspace_given || nspace_in_enviro) { - (void)strncpy(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN); + pmix_strncpy(pmix_globals.myid.nspace, nspace, PMIX_MAX_NSLEN); free(nspace); pmix_globals.myid.rank = rank; } @@ -452,7 +464,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_NOMEM; } - pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_nspace_t); + pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); if (NULL == pmix_client_globals.myserver->nptr) { PMIX_RELEASE(pmix_client_globals.myserver); if (gdsfound) { @@ -582,7 +594,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, } if (!nspace_given) { /* Success, so copy the nspace and rank to the proc struct they gave us */ - (void)strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(proc->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); } if (!rank_given) { proc->rank = pmix_globals.myid.rank; @@ -600,36 +612,21 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, pmix_globals.mypeer->info->pname.nspace = strdup(pmix_globals.myid.nspace); pmix_globals.mypeer->info->pname.rank = pmix_globals.myid.rank; - /* if we are acting as a client, then send a request for our - * job info - we do this as a non-blocking - * transaction because some systems cannot handle very large - * blocking operations and error out if we try them. */ - if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { - req = PMIX_NEW(pmix_buffer_t); - PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, - req, &cmd, 1, PMIX_COMMAND); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(req); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - /* send to the server */ - PMIX_CONSTRUCT(&cb, pmix_cb_t); - PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, - req, job_data, (void*)&cb); - if (PMIX_SUCCESS != rc) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - /* wait for the data to return */ - PMIX_WAIT_THREAD(&cb.lock); - rc = cb.status; - PMIX_DESTRUCT(&cb); - if (PMIX_SUCCESS != rc) { + /* if we are acting as a server, then start listening */ + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + /* setup the wildcard recv for inbound messages from clients */ + rcv = PMIX_NEW(pmix_ptl_posted_recv_t); + rcv->tag = UINT32_MAX; + rcv->cbfunc = pmix_server_message_handler; + /* add it to the end of the list of recvs */ + pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super); + /* open the pnet framework so we can harvest envars */ + rc = pmix_mca_base_framework_open(&pmix_pnet_base_framework, 0); + if (PMIX_SUCCESS != rc){ PMIX_RELEASE_THREAD(&pmix_global_lock); return rc; } + /* note that we do not select active plugins as we don't need them */ } /* setup IOF */ @@ -678,7 +675,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, &stdinev.ev, fd, PMIX_EV_READ, pmix_iof_read_local_handler, &stdinev); - } \ + } /* check to see if we want the stdin read event to be * active - we will always at least define the event, * but may delay its activation @@ -711,12 +708,42 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, /* increment our init reference counter */ pmix_globals.init_cntr++; - if (!PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + /* if we are acting as a client, then send a request for our + * job info - we do this as a non-blocking + * transaction because some systems cannot handle very large + * blocking operations and error out if we try them. */ + if (PMIX_PROC_IS_CLIENT(pmix_globals.mypeer)) { + req = PMIX_NEW(pmix_buffer_t); + PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, + req, &cmd, 1, PMIX_COMMAND); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(req); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* send to the server */ + PMIX_CONSTRUCT(&cb, pmix_cb_t); + PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver, + req, job_data, (void*)&cb); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* wait for the data to return */ + PMIX_WAIT_THREAD(&cb.lock); + rc = cb.status; + PMIX_DESTRUCT(&cb); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + } else { /* now finish the initialization by filling our local * datastore with typical job-related info. No point * in having the server generate these as we are * obviously a singleton, and so the values are well-known */ - (void)strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); wildcard.rank = pmix_globals.myid.rank; /* the jobid is just our nspace */ @@ -1024,13 +1051,6 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, /* if we are acting as a server, then start listening */ if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { - /* setup the wildcard recv for inbound messages from clients */ - rcv = PMIX_NEW(pmix_ptl_posted_recv_t); - rcv->tag = UINT32_MAX; - rcv->cbfunc = pmix_server_message_handler; - /* add it to the end of the list of recvs */ - pmix_list_append(&pmix_ptl_globals.posted_recvs, &rcv->super); - /* start listening for connections */ if (PMIX_SUCCESS != pmix_ptl_base_start_listening(info, ninfo)) { pmix_show_help("help-pmix-server.txt", "listener-thread-start", true); @@ -1073,8 +1093,8 @@ static void finwait_cbfunc(struct pmix_peer_t *pr, if (tev->active) { tev->active = false; pmix_event_del(&tev->ev); // stop the timer - PMIX_WAKEUP_THREAD(&tev->lock); } + PMIX_WAKEUP_THREAD(&tev->lock); } PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) @@ -1083,10 +1103,9 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) pmix_cmd_t cmd = PMIX_FINALIZE_CMD; pmix_status_t rc; pmix_tool_timeout_t tev; - struct timeval tv = {2, 0}; + struct timeval tv = {5, 0}; int n; pmix_peer_t *peer; - pmix_setup_caddy_t *cd; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (1 != pmix_globals.init_cntr) { @@ -1095,6 +1114,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) return PMIX_SUCCESS; } pmix_globals.init_cntr = 0; + pmix_globals.mypeer->finalized = true; PMIX_RELEASE_THREAD(&pmix_global_lock); pmix_output_verbose(2, pmix_globals.debug_output, @@ -1142,6 +1162,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) /* wait for the ack to return */ PMIX_WAIT_THREAD(&tev.lock); PMIX_DESTRUCT_LOCK(&tev.lock); + if (tev.active) { pmix_event_del(&tev.ev); } @@ -1158,7 +1179,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) (void)pmix_progress_thread_pause(NULL); } - PMIX_RELEASE(pmix_client_globals.myserver); +// PMIX_RELEASE(pmix_client_globals.myserver); PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests); for (n=0; n < pmix_client_globals.peers.size; n++) { if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_client_globals.peers, n))) { @@ -1169,19 +1190,13 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { pmix_ptl_base_stop_listening(); - /* cleanout any IOF */ - for (n=0; n < PMIX_IOF_HOTEL_SIZE; n++) { - pmix_hotel_checkout_and_return_occupant(&pmix_server_globals.iof, n, (void**)&cd); - if (NULL != cd) { - PMIX_RELEASE(cd); - } - } - PMIX_DESTRUCT(&pmix_server_globals.iof); for (n=0; n < pmix_server_globals.clients.size; n++) { if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, n))) { PMIX_RELEASE(peer); } } + + (void)pmix_mca_base_framework_close(&pmix_pnet_base_framework); PMIX_DESTRUCT(&pmix_server_globals.clients); PMIX_LIST_DESTRUCT(&pmix_server_globals.collectives); PMIX_LIST_DESTRUCT(&pmix_server_globals.remote_pnd); @@ -1189,6 +1204,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) PMIX_LIST_DESTRUCT(&pmix_server_globals.gdata); PMIX_LIST_DESTRUCT(&pmix_server_globals.events); PMIX_LIST_DESTRUCT(&pmix_server_globals.nspaces); + PMIX_LIST_DESTRUCT(&pmix_server_globals.iof); } /* shutdown services */ @@ -1199,6 +1215,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) /* finalize the class/object system */ pmix_class_finalize(); + return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pevent/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/tools/pevent/Makefile.am index 92fce9ac2d..10e2b321a9 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pevent/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pevent/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2008-2014 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -29,4 +29,5 @@ endif # PMIX_INSTALL_BINARIES pevent_SOURCES = pevent.c pevent_LDADD = \ + $(PMIX_EXTRA_LTLIB) \ $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pevent/pevent.c b/opal/mca/pmix/pmix3x/pmix/src/tools/pevent/pevent.c index f472e78ed4..a5eccb8666 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pevent/pevent.c +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pevent/pevent.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -23,7 +23,9 @@ * */ -#define _GNU_SOURCE +#include "pmix_config.h" +#include "pmix_common.h" + #include #include #include diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/plookup/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/tools/plookup/Makefile.am index a273ea65d8..bf7a64d5c4 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/plookup/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/plookup/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2008-2014 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -29,4 +29,5 @@ endif # PMIX_INSTALL_BINARIES plookup_SOURCES = plookup.c plookup_LDADD = \ - $(top_builddir)/src/libpmix.la + $(PMIX_EXTRA_LTLIB) \ + $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/plookup/plookup.c b/opal/mca/pmix/pmix3x/pmix/src/tools/plookup/plookup.c index 3c8cccac2b..d206ace840 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/plookup/plookup.c +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/plookup/plookup.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -269,7 +269,7 @@ int main(int argc, char **argv) /* setup the keys */ PMIX_PDATA_CREATE(pdata, ndata); for (n=0; n < ndata; n++) { - (void)strncpy(pdata[n].key, keys[n], PMIX_MAX_KEYLEN); + pmix_strncpy(pdata[n].key, keys[n], PMIX_MAX_KEYLEN); } /* perform the lookup */ rc = PMIx_Lookup(pdata, ndata, info, ninfo); diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/Makefile.am index 8fe2640c7a..343cfaa3b8 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pmix_info/Makefile.am @@ -31,7 +31,6 @@ AM_CFLAGS = \ -DPMIX_BUILD_LDFLAGS="\"@LDFLAGS@\"" \ -DPMIX_BUILD_LIBS="\"@LIBS@\"" \ -DPMIX_CC_ABSOLUTE="\"@PMIX_CC_ABSOLUTE@\"" \ - -DPMIX_CONFIGURE_CLI="\"@PMIX_CONFIGURE_CLI@\"" \ -DPMIX_GREEK_VERSION="\"@PMIX_GREEK_VERSION@\"" \ -DPMIX_REPO_REV="\"@PMIX_REPO_REV@\"" \ -DPMIX_RELEASE_DATE="\"@PMIX_RELEASE_DATE@\"" @@ -51,4 +50,5 @@ pmix_info_SOURCES = \ support.c pmix_info_LDADD = \ - $(top_builddir)/src/libpmix.la + $(PMIX_EXTRA_LTLIB) \ + $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix3x/pmix/src/tools/pps/Makefile.am b/opal/mca/pmix/pmix3x/pmix/src/tools/pps/Makefile.am index 930a4672e9..ac7bc9eb42 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/tools/pps/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/src/tools/pps/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2008-2014 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. -# Copyright (c) 2017 Intel, Inc. All rights reserved. +# Copyright (c) 2017-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -29,4 +29,5 @@ endif # PMIX_INSTALL_BINARIES pps_SOURCES = pps.c pps_LDADD = \ - $(top_builddir)/src/libpmix.la + $(PMIX_EXTRA_LTLIB) \ + $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/argv.c b/opal/mca/pmix/pmix3x/pmix/src/util/argv.c index f5c08f80a0..8eb1e3f6dc 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/argv.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/argv.c @@ -11,7 +11,7 @@ * All rights reserved. * Copyright (c) 2007 Voltaire. All rights reserved. * Copyright (c) 2012 Los Alamos National Security, LLC. All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -221,7 +221,7 @@ static char **pmix_argv_split_inter(const char *src_string, int delimiter, if (NULL == argtemp) return NULL; - strncpy(argtemp, src_string, arglen); + pmix_strncpy(argtemp, src_string, arglen); argtemp[arglen] = '\0'; if (PMIX_SUCCESS != pmix_argv_append(&argc, &argv, argtemp)) { @@ -235,7 +235,7 @@ static char **pmix_argv_split_inter(const char *src_string, int delimiter, /* short argument, copy to buffer and add */ else { - strncpy(arg, src_string, arglen); + pmix_strncpy(arg, src_string, arglen); arg[arglen] = '\0'; if (PMIX_SUCCESS != pmix_argv_append(&argc, &argv, arg)) diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/basename.c b/opal/mca/pmix/pmix3x/pmix/src/util/basename.c index 64e5c27e7e..aa2076d7ea 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/basename.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/basename.c @@ -12,7 +12,7 @@ * Copyright (c) 2009-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2014-2015 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -123,11 +123,7 @@ char* pmix_dirname(const char* filename) } if( p != filename ) { char* ret = (char*)malloc( p - filename + 1 ); -#ifdef HAVE_STRNCPY_S - strncpy_s( ret, (p - filename + 1), filename, p - filename ); -#else - strncpy(ret, filename, p - filename); -#endif + pmix_strncpy(ret, filename, p - filename); ret[p - filename] = '\0'; return pmix_make_filename_os_friendly(ret); } diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/error.c b/opal/mca/pmix/pmix3x/pmix/src/util/error.c index fddf8e4b67..7930be1c33 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/error.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/error.c @@ -159,28 +159,24 @@ PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t errnum) return "LOST-PEER-CONNECTION"; case PMIX_ERR_LOST_CONNECTION_TO_CLIENT: return "LOST-CONNECTION-TO-CLIENT"; - - case PMIX_QUERY_PARTIAL_SUCCESS: return "QUERY-PARTIAL-SUCCESS"; - - case PMIX_NOTIFY_ALLOC_COMPLETE: return "PMIX ALLOC OPERATION COMPLETE"; - - case PMIX_JCTRL_CHECKPOINT: return "PMIX JOB CONTROL CHECKPOINT"; case PMIX_JCTRL_CHECKPOINT_COMPLETE: return "PMIX JOB CONTROL CHECKPOINT COMPLETE"; case PMIX_JCTRL_PREEMPT_ALERT: return "PMIX PRE-EMPTION ALERT"; - - case PMIX_MONITOR_HEARTBEAT_ALERT: return "PMIX HEARTBEAT ALERT"; case PMIX_MONITOR_FILE_ALERT: return "PMIX FILE MONITOR ALERT"; + case PMIX_PROC_TERMINATED: + return "PROC-TERMINATED"; + case PMIX_ERR_INVALID_TERMINATION: + return "INVALID-TERMINATION"; case PMIX_ERR_EVENT_REGISTRATION: return "EVENT-REGISTRATION"; @@ -196,20 +192,28 @@ PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t errnum) return "PROC-HAS-CONNECTED"; case PMIX_CONNECT_REQUESTED: return "CONNECT-REQUESTED"; + case PMIX_OPENMP_PARALLEL_ENTERED: + return "OPENMP-PARALLEL-ENTERED"; + case PMIX_OPENMP_PARALLEL_EXITED: + return "OPENMP-PARALLEL-EXITED"; + case PMIX_LAUNCH_DIRECTIVE: return "LAUNCH-DIRECTIVE"; case PMIX_LAUNCHER_READY: return "LAUNCHER-READY"; case PMIX_OPERATION_IN_PROGRESS: return "OPERATION-IN-PROGRESS"; - case PMIX_PROC_TERMINATED: - return "PROC-TERMINATED"; + case PMIX_OPERATION_SUCCEEDED: + return "OPERATION-SUCCEEDED"; + case PMIX_ERR_INVALID_OPERATION: + return "INVALID-OPERATION"; case PMIX_ERR_NODE_DOWN: return "NODE-DOWN"; case PMIX_ERR_NODE_OFFLINE: return "NODE-OFFLINE"; - + case PMIX_ERR_SYS_OTHER: + return "UNDEFINED-SYSTEM-EVENT"; case PMIX_EVENT_NO_ACTION_TAKEN: return "EVENT-NO-ACTION-TAKEN"; diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/hash.c b/opal/mca/pmix/pmix3x/pmix/src/util/hash.c index 1a0a95744e..806781186b 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/hash.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/hash.c @@ -6,7 +6,7 @@ * reserved. * Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2016 Mellanox Technologies, Inc. @@ -172,7 +172,7 @@ pmix_status_t pmix_hash_fetch(pmix_hash_table_t *table, pmix_rank_t rank, /* copy the list elements */ n=0; PMIX_LIST_FOREACH(hv, &proc_data->data, pmix_kval_t) { - (void)strncpy(info[n].key, hv->key, PMIX_MAX_KEYLEN); + pmix_strncpy(info[n].key, hv->key, PMIX_MAX_KEYLEN); pmix_value_xfer(&info[n].value, hv->value); ++n; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/keyval_parse.c b/opal/mca/pmix/pmix3x/pmix/src/util/keyval_parse.c index c07e65e668..52c68fa50d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/keyval_parse.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/keyval_parse.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -136,7 +136,7 @@ static int parse_line(void) key_buffer = tmp; } - strncpy(key_buffer, pmix_util_keyval_yytext, key_buffer_len); + pmix_strncpy(key_buffer, pmix_util_keyval_yytext, key_buffer_len-1); /* The first thing we have to see is an "=" */ @@ -259,7 +259,7 @@ static int save_param_name (void) key_buffer = tmp; } - strncpy (key_buffer, pmix_util_keyval_yytext, key_buffer_len); + pmix_strncpy (key_buffer, pmix_util_keyval_yytext, key_buffer_len-1); return PMIX_SUCCESS; } diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/net.c b/opal/mca/pmix/pmix3x/pmix/src/util/net.c index 22baf7d5a4..34661df3f2 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/net.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/net.c @@ -50,23 +50,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/os_path.c b/opal/mca/pmix/pmix3x/pmix/src/util/os_path.c index 6abedaf8a3..3f94f9bb7d 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/os_path.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/os_path.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -34,7 +34,7 @@ static const char *path_sep = PMIX_PATH_SEP; -char *pmix_os_path(bool relative, ...) +char *pmix_os_path(int relative, ...) { va_list ap; char *element, *path; diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/os_path.h b/opal/mca/pmix/pmix3x/pmix/src/util/os_path.h index 9f3c71bfb5..3933d04630 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/os_path.h +++ b/opal/mca/pmix/pmix3x/pmix/src/util/os_path.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -64,8 +64,13 @@ BEGIN_C_DECLS * provided path elements, separated by the path separator character * appropriate to the local operating system. The path_name string has been malloc'd * and therefore the user is responsible for free'ing the field. + * + * Note that the "relative" argument is int instead of bool, because + * passing a parameter that undergoes default argument promotion to + * va_start() has undefined behavior (according to clang warnings on + * MacOS High Sierra). */ -PMIX_EXPORT char *pmix_os_path(bool relative, ...) __pmix_attribute_malloc__ __pmix_attribute_sentinel__ __pmix_attribute_warn_unused_result__; +PMIX_EXPORT char *pmix_os_path(int relative, ...) __pmix_attribute_malloc__ __pmix_attribute_sentinel__ __pmix_attribute_warn_unused_result__; /** * Convert the path to be OS friendly. On UNIX this function will diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/output.c b/opal/mca/pmix/pmix3x/pmix/src/util/output.c index 1d3d4148dc..8648f1a0b7 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/output.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/output.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -694,7 +694,7 @@ static int open_file(int i) if (NULL == filename) { return PMIX_ERR_OUT_OF_RESOURCE; } - strncpy(filename, output_dir, PMIX_PATH_MAX); + pmix_strncpy(filename, output_dir, PMIX_PATH_MAX-1); strcat(filename, "/"); if (NULL != output_prefix) { strcat(filename, output_prefix); diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/pif.c b/opal/mca/pmix/pmix3x/pmix/src/util/pif.c index 78ca9559ff..9696502f4b 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/pif.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/pif.c @@ -16,7 +16,7 @@ * reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -51,23 +51,7 @@ #include #endif #ifdef HAVE_NET_IF_H -#if defined(__APPLE__) && defined(_LP64) -/* Apple engineering suggested using options align=power as a - workaround for a bug in OS X 10.4 (Tiger) that prevented ioctl(..., - SIOCGIFCONF, ...) from working properly in 64 bit mode on Power PC. - It turns out that the underlying issue is the size of struct - ifconf, which the kernel expects to be 12 and natural 64 bit - alignment would make 16. The same bug appears in 64 bit mode on - Intel macs, but align=power is a no-op there, so instead, use the - pack pragma to instruct the compiler to pack on 4 byte words, which - has the same effect as align=power for our needs and works on both - Intel and Power PC Macs. */ -#pragma pack(push,4) -#endif #include -#if defined(__APPLE__) && defined(_LP64) -#pragma pack(pop) -#endif #endif #ifdef HAVE_NETDB_H #include @@ -218,7 +202,7 @@ int pmix_ifaddrtoname(const char* if_addr, char* if_name, int length) memcpy (&ipv4, r->ai_addr, r->ai_addrlen); if (inaddr->sin_addr.s_addr == ipv4.sin_addr.s_addr) { - strncpy(if_name, intf->if_name, length); + pmix_strncpy(if_name, intf->if_name, length-1); freeaddrinfo (res); return PMIX_SUCCESS; } @@ -226,7 +210,7 @@ int pmix_ifaddrtoname(const char* if_addr, char* if_name, int length) else { if (IN6_ARE_ADDR_EQUAL(&((struct sockaddr_in6*) &intf->if_addr)->sin6_addr, &((struct sockaddr_in6*) r->ai_addr)->sin6_addr)) { - strncpy(if_name, intf->if_name, length); + pmix_strncpy(if_name, intf->if_name, length-1); freeaddrinfo (res); return PMIX_SUCCESS; } @@ -493,7 +477,7 @@ int pmix_ifindextoname(int if_index, char* if_name, int length) intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list); intf = (pmix_pif_t*)pmix_list_get_next(intf)) { if (intf->if_index == if_index) { - strncpy(if_name, intf->if_name, length); + pmix_strncpy(if_name, intf->if_name, length-1); return PMIX_SUCCESS; } } @@ -514,7 +498,7 @@ int pmix_ifkindextoname(int if_kindex, char* if_name, int length) intf != (pmix_pif_t*)pmix_list_get_end(&pmix_if_list); intf = (pmix_pif_t*)pmix_list_get_next(intf)) { if (intf->if_kernel_index == if_kindex) { - strncpy(if_name, intf->if_name, length); + pmix_strncpy(if_name, intf->if_name, length-1); return PMIX_SUCCESS; } } diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/show_help.c b/opal/mca/pmix/pmix3x/pmix/src/util/show_help.c index 1654d39ead..ae95ac691b 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/show_help.c +++ b/opal/mca/pmix/pmix3x/pmix/src/util/show_help.c @@ -12,7 +12,7 @@ * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -49,9 +49,9 @@ static char **search_dirs = NULL; * Local functions */ static int pmix_show_vhelp_internal(const char *filename, const char *topic, - bool want_error_header, va_list arglist); + int want_error_header, va_list arglist); static int pmix_show_help_internal(const char *filename, const char *topic, - bool want_error_header, ...); + int want_error_header, ...); pmix_show_help_fn_t pmix_show_help = pmix_show_help_internal; pmix_show_vhelp_fn_t pmix_show_vhelp = pmix_show_vhelp_internal; @@ -90,7 +90,7 @@ int pmix_show_help_finalize(void) * not optimization. :-) */ static int array2string(char **outstring, - bool want_error_header, char **lines) + int want_error_header, char **lines) { int i, count; size_t len; @@ -298,7 +298,7 @@ static int load_array(char ***array, const char *filename, const char *topic) } char *pmix_show_help_vstring(const char *filename, const char *topic, - bool want_error_header, va_list arglist) + int want_error_header, va_list arglist) { int rc; char *single_string, *output, **array = NULL; @@ -324,7 +324,7 @@ char *pmix_show_help_vstring(const char *filename, const char *topic, } char *pmix_show_help_string(const char *filename, const char *topic, - bool want_error_handler, ...) + int want_error_handler, ...) { char *output; va_list arglist; @@ -338,7 +338,7 @@ char *pmix_show_help_string(const char *filename, const char *topic, } static int pmix_show_vhelp_internal(const char *filename, const char *topic, - bool want_error_header, va_list arglist) + int want_error_header, va_list arglist) { char *output; @@ -356,7 +356,7 @@ static int pmix_show_vhelp_internal(const char *filename, const char *topic, } static int pmix_show_help_internal(const char *filename, const char *topic, - bool want_error_header, ...) + int want_error_header, ...) { va_list arglist; int rc; diff --git a/opal/mca/pmix/pmix3x/pmix/src/util/show_help.h b/opal/mca/pmix/pmix3x/pmix/src/util/show_help.h index 8c23887dc3..1129a762a7 100644 --- a/opal/mca/pmix/pmix3x/pmix/src/util/show_help.h +++ b/opal/mca/pmix/pmix3x/pmix/src/util/show_help.h @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -129,9 +129,14 @@ PMIX_EXPORT int pmix_show_help_finalize(void); * (typically $prefix/share/pmix), and looks up the message * based on the topic, and displays it. If want_error_header is * true, a header and footer of asterisks are also displayed. + * + * Note that the "want_error_header" argument is int instead of bool, + * because passing a parameter that undergoes default argument + * promotion to va_start() has undefined behavior (according to clang + * warnings on MacOS High Sierra). */ typedef int (*pmix_show_help_fn_t)(const char *filename, const char *topic, - bool want_error_header, ...); + int want_error_header, ...); PMIX_EXPORT extern pmix_show_help_fn_t pmix_show_help; /** @@ -139,7 +144,7 @@ PMIX_EXPORT extern pmix_show_help_fn_t pmix_show_help; * a va_list form of varargs. */ typedef int (*pmix_show_vhelp_fn_t)(const char *filename, const char *topic, - bool want_error_header, va_list ap); + int want_error_header, va_list ap); PMIX_EXPORT extern pmix_show_vhelp_fn_t pmix_show_vhelp; /** @@ -148,7 +153,7 @@ PMIX_EXPORT extern pmix_show_vhelp_fn_t pmix_show_vhelp; */ PMIX_EXPORT char* pmix_show_help_string(const char *filename, const char *topic, - bool want_error_header, ...); + int want_error_header, ...); /** * This function does the same thing as pmix_show_help_string(), but @@ -156,7 +161,7 @@ PMIX_EXPORT char* pmix_show_help_string(const char *filename, */ PMIX_EXPORT char* pmix_show_help_vstring(const char *filename, const char *topic, - bool want_error_header, va_list ap); + int want_error_header, va_list ap); /** * This function adds another search location for the files that diff --git a/opal/mca/pmix/pmix3x/pmix/test/Makefile.am b/opal/mca/pmix/pmix3x/pmix/test/Makefile.am index 64ad119878..3b4ee7214c 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/Makefile.am +++ b/opal/mca/pmix/pmix3x/pmix/test/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2013-2018 Intel, Inc. All rights reserved. # Copyright (c) 2018 Research Organization for Information Science # and Technology (RIST). All rights reserved. # $COPYRIGHT$ @@ -29,7 +29,7 @@ endif headers = test_common.h cli_stages.h server_callbacks.h utils.h test_fence.h \ test_publish.h test_spawn.h test_cd.h test_resolve_peers.h test_error.h \ - test_replace.h test_internal.h + test_replace.h test_internal.h test_server.h AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/src/api @@ -43,7 +43,7 @@ endif noinst_PROGRAMS += pmix_test pmix_client pmix_regex pmix_test_SOURCES = $(headers) \ - pmix_test.c test_common.c cli_stages.c server_callbacks.c utils.c + pmix_test.c test_common.c cli_stages.c server_callbacks.c test_server.c utils.c pmix_test_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) pmix_test_LDADD = \ $(top_builddir)/src/libpmix.la @@ -70,7 +70,7 @@ pmix_client_LDADD = \ $(top_builddir)/src/libpmix.la pmix_regex_SOURCES = $(headers) \ - pmix_regex.c test_common.c cli_stages.c server_callbacks.c utils.c + pmix_regex.c test_common.c cli_stages.c server_callbacks.c test_server.c utils.c pmix_regex_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) pmix_regex_LDADD = \ $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix3x/pmix/test/cli_stages.c b/opal/mca/pmix/pmix3x/pmix/test/cli_stages.c index 04049ad9cc..ecd41c2bd4 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/cli_stages.c +++ b/opal/mca/pmix/pmix3x/pmix/test/cli_stages.c @@ -3,6 +3,8 @@ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. + * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -21,18 +23,27 @@ int cli_rank(cli_info_t *cli) int i; for(i=0; i < cli_info_cnt; i++){ if( cli == &cli_info[i] ){ - return i; + return cli->rank; } } return -1; } -void cli_init(int nprocs, cli_state_t order[]) +void cli_init(int nprocs) { int n, i; + cli_state_t order[CLI_TERM+1]; + cli_info = malloc( sizeof(cli_info_t) * nprocs); cli_info_cnt = nprocs; + order[CLI_UNINIT] = CLI_FORKED; + order[CLI_FORKED] = CLI_FIN; + order[CLI_CONNECTED] = CLI_UNDEF; + order[CLI_FIN] = CLI_TERM; + order[CLI_DISCONN] = CLI_UNDEF; + order[CLI_TERM] = CLI_UNDEF; + for (n=0; n < nprocs; n++) { cli_info[n].sd = -1; cli_info[n].ev = NULL; @@ -198,8 +209,9 @@ void cli_wait_all(double timeout) TEST_VERBOSE(("waitpid = %d", pid)); for(i=0; i < cli_info_cnt; i++){ if( cli_info[i].pid == pid ){ - TEST_VERBOSE(("the child with pid = %d has rank = %d\n" - "\t\texited = %d, signalled = %d", pid, i, + TEST_VERBOSE(("the child with pid = %d has rank = %d, ns = %s\n" + "\t\texited = %d, signalled = %d", pid, + cli_info[i].rank, cli_info[i].ns, WIFEXITED(status), WIFSIGNALED(status) )); if( WIFEXITED(status) || WIFSIGNALED(status) ){ cli_cleanup(&cli_info[i]); @@ -211,6 +223,9 @@ void cli_wait_all(double timeout) if( errno == ECHILD ){ TEST_VERBOSE(("No more children to wait. Happens on the last cli_wait_all call " "which is used to ensure that all children terminated.\n")); + if (pmix_test_verbose) { + sleep(1); + } break; } else { TEST_ERROR(("waitpid(): %d : %s", errno, strerror(errno))); diff --git a/opal/mca/pmix/pmix3x/pmix/test/cli_stages.h b/opal/mca/pmix/pmix3x/pmix/test/cli_stages.h index 343af2de04..b716480b0a 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/cli_stages.h +++ b/opal/mca/pmix/pmix3x/pmix/test/cli_stages.h @@ -3,6 +3,8 @@ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. + * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -10,6 +12,9 @@ * $HEADER$ */ +#ifndef CLI_STAGES_H +#define CLI_STAGES_H + #include #include #include @@ -41,7 +46,7 @@ typedef struct { pmix_event_t *ev; cli_state_t state; cli_state_t next_state[CLI_TERM+1]; - int rank; + pmix_rank_t rank; char *ns; } cli_info_t; @@ -50,7 +55,7 @@ extern int cli_info_cnt; extern bool test_abort; int cli_rank(cli_info_t *cli); -void cli_init(int nprocs, cli_state_t order[]); +void cli_init(int nprocs); void cli_connect(cli_info_t *cli, int sd, struct event_base * ebase, event_callback_fn callback); void cli_finalize(cli_info_t *cli); void cli_disconnect(cli_info_t *cli); @@ -75,3 +80,5 @@ void op_callbk(pmix_status_t status, void errhandler_reg_callbk (pmix_status_t status, size_t errhandler_ref, void *cbdata); + +#endif // CLI_STAGES_H diff --git a/opal/mca/pmix/pmix3x/pmix/test/pmix_client.c b/opal/mca/pmix/pmix3x/pmix/test/pmix_client.c index e00a44e22e..3d1b46fcfa 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/pmix_client.c +++ b/opal/mca/pmix/pmix3x/pmix/test/pmix_client.c @@ -13,8 +13,8 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015-2017 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -78,7 +78,7 @@ int main(int argc, char **argv) parse_cmd(argc, argv, ¶ms); // We don't know rank at this place! - TEST_VERBOSE(("Client ns %s rank %d: Start", params.nspace, params.rank)); + TEST_VERBOSE(("Client %s:%d started PID:%d", params.nspace, params.rank, getpid())); /* handle early-fail test case */ if (1 == params.early_fail && 0 == params.rank) { @@ -236,6 +236,7 @@ int main(int argc, char **argv) TEST_VERBOSE(("Client ns %s rank %d:PMIx_Finalize successfully completed", myproc.nspace, myproc.rank)); } + TEST_VERBOSE(("Client %s:%d finished PID:%d", params.nspace, params.rank, getpid())); TEST_OUTPUT_CLEAR(("OK\n")); TEST_CLOSE_FILE(); FREE_TEST_PARAMS(params); diff --git a/opal/mca/pmix/pmix3x/pmix/test/pmix_test.c b/opal/mca/pmix/pmix3x/pmix/test/pmix_test.c index c1a8130b1a..9ceeb72d53 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/pmix_test.c +++ b/opal/mca/pmix/pmix3x/pmix/test/pmix_test.c @@ -13,11 +13,12 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. * All rights reserved. + * * $COPYRIGHT$ * * Additional copyrights may follow @@ -35,7 +36,8 @@ #include "server_callbacks.h" #include "utils.h" -#include "src/include/pmix_globals.h" +#include "test_server.h" +#include "test_common.h" bool spawn_wait = false; @@ -47,7 +49,6 @@ int main(int argc, char **argv) struct stat stat_buf; struct timeval tv; double test_start; - cli_state_t order[CLI_TERM+1]; test_params params; INIT_TEST_PARAMS(params); int test_fail = 0; @@ -68,6 +69,14 @@ int main(int argc, char **argv) parse_cmd(argc, argv, ¶ms); TEST_VERBOSE(("Start PMIx_lite smoke test (timeout is %d)", params.timeout)); + /* set common argv and env */ + client_env = pmix_argv_copy(environ); + set_client_argv(¶ms, &client_argv); + + tmp = pmix_argv_join(client_argv, ' '); + TEST_VERBOSE(("Executing test: %s", tmp)); + free(tmp); + /* verify executable */ if( 0 > ( rc = stat(params.binary, &stat_buf) ) ){ TEST_ERROR(("Cannot stat() executable \"%s\": %d: %s", params.binary, errno, strerror(errno))); @@ -83,48 +92,29 @@ int main(int argc, char **argv) return 0; } - /* setup the server library */ - pmix_info_t info[1]; - (void)strncpy(info[0].key, PMIX_SOCKET_MODE, PMIX_MAX_KEYLEN); - info[0].value.type = PMIX_UINT32; - info[0].value.data.uint32 = 0666; - - if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, 1))) { - TEST_ERROR(("Init failed with error %d", rc)); + if (PMIX_SUCCESS != (rc = server_init(¶ms))) { FREE_TEST_PARAMS(params); return rc; } - /* register the errhandler */ - PMIx_Register_event_handler(NULL, 0, NULL, 0, - errhandler, errhandler_reg_callbk, NULL); - order[CLI_UNINIT] = CLI_FORKED; - order[CLI_FORKED] = CLI_FIN; - order[CLI_CONNECTED] = CLI_UNDEF; - order[CLI_FIN] = CLI_TERM; - order[CLI_DISCONN] = CLI_UNDEF; - order[CLI_TERM] = CLI_UNDEF; - cli_init(params.nprocs, order); - - /* set common argv and env */ - client_env = pmix_argv_copy(environ); - set_client_argv(¶ms, &client_argv); - - tmp = pmix_argv_join(client_argv, ' '); - TEST_VERBOSE(("Executing test: %s", tmp)); - free(tmp); + cli_init(params.lsize); int launched = 0; /* set namespaces and fork clients */ if (NULL == params.ns_dist) { + uint32_t i; + int base_rank = 0; + + /* compute my start counter */ + for(i = 0; i < (uint32_t)my_server_id; i++) { + base_rank += (params.nprocs % params.nservers) > (uint32_t)i ? + params.nprocs / params.nservers + 1 : + params.nprocs / params.nservers; + } /* we have a single namespace for all clients */ ns_nprocs = params.nprocs; - rc = launch_clients(ns_nprocs, params.binary, &client_env, &client_argv); - if (PMIX_SUCCESS != rc) { - FREE_TEST_PARAMS(params); - return rc; - } - launched += ns_nprocs; + launched += server_launch_clients(params.lsize, params.nprocs, base_rank, + ¶ms, &client_env, &client_argv); } else { char *pch; pch = strtok(params.ns_dist, ":"); @@ -136,17 +126,13 @@ int main(int argc, char **argv) return PMIX_ERROR; } if (0 < ns_nprocs) { - rc = launch_clients(ns_nprocs, params.binary, &client_env, &client_argv); - if (PMIX_SUCCESS != rc) { - FREE_TEST_PARAMS(params); - return rc; - } + launched += server_launch_clients(ns_nprocs, ns_nprocs, 0, ¶ms, + &client_env, &client_argv); } pch = strtok (NULL, ":"); - launched += ns_nprocs; } } - if (params.nprocs != (uint32_t)launched) { + if (params.lsize != (uint32_t)launched) { TEST_ERROR(("Total number of processes doesn't correspond number specified by ns_dist parameter.")); cli_kill_all(); test_fail = 1; @@ -185,24 +171,16 @@ int main(int argc, char **argv) PMIX_WAIT_FOR_COMPLETION(spawn_wait); } - pmix_argv_free(client_argv); - pmix_argv_free(client_env); - /* deregister the errhandler */ PMIx_Deregister_event_handler(0, op_callbk, NULL); cli_wait_all(1.0); - /* finalize the server library */ - if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) { - TEST_ERROR(("Finalize failed with error %d", rc)); - } + test_fail += server_finalize(¶ms); FREE_TEST_PARAMS(params); - - if (0 == test_fail) { - TEST_OUTPUT(("Test finished OK!")); - } + pmix_argv_free(client_argv); + pmix_argv_free(client_env); return test_fail; } diff --git a/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c b/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c index 783d45d628..ae16129ecf 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c +++ b/opal/mca/pmix/pmix3x/pmix/test/server_callbacks.c @@ -2,7 +2,7 @@ * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ @@ -17,6 +17,7 @@ #include #include "server_callbacks.h" #include "src/util/argv.h" +#include "test_server.h" extern bool spawn_wait; @@ -36,28 +37,6 @@ pmix_server_module_t mymodule = { .deregister_events = deregevents_fn }; -typedef struct { - pmix_list_item_t super; - pmix_modex_data_t data; -} pmix_test_data_t; - -static void pcon(pmix_test_data_t *p) -{ - p->data.blob = NULL; - p->data.size = 0; -} - -static void pdes(pmix_test_data_t *p) -{ - if (NULL != p->data.blob) { - free(p->data.blob); - } -} - -PMIX_CLASS_INSTANCE(pmix_test_data_t, - pmix_list_item_t, - pcon, pdes); - typedef struct { pmix_list_item_t super; pmix_info_t data; @@ -95,12 +74,25 @@ pmix_status_t connected(const pmix_proc_t *proc, void *server_object, pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { - if( CLI_TERM <= cli_info[proc->rank].state ){ + cli_info_t *cli = NULL; + int i; + for (i = 0; i < cli_info_cnt; i++) { + if((proc->rank == cli_info[i].rank) && + (0 == strcmp(proc->nspace, cli_info[i].ns))){ + cli = &cli_info[i]; + break; + } + } + if (NULL == cli) { + TEST_ERROR(("cannot found rank %d", proc->rank)); + return PMIX_SUCCESS; + } + if( CLI_TERM <= cli->state ){ TEST_ERROR(("double termination of rank %d", proc->rank)); return PMIX_SUCCESS; } - TEST_VERBOSE(("Rank %d terminated", proc->rank)); - cli_finalize(&cli_info[proc->rank]); + TEST_VERBOSE(("Rank %s:%d terminated", proc->nspace, proc->rank)); + cli_finalize(cli); finalized_count++; if (finalized_count == cli_info_cnt) { if (NULL != pmix_test_published_list) { @@ -135,16 +127,13 @@ pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs, TEST_VERBOSE(("Getting data for %s:%d", procs[0].nspace, procs[0].rank)); - /* In a perfect world, we should wait until - * the test servers from all involved procs - * respond. We don't have multi-server capability - * yet, so we'll just respond right away and - * return what we were given */ - - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, data, ndata, cbdata, NULL, NULL); + if ((pmix_list_get_size(server_list) == 1) && (my_server_id == 0)) { + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, data, ndata, cbdata, NULL, NULL); + } + return PMIX_SUCCESS; } - return PMIX_SUCCESS; + return server_fence_contrib(data, ndata, cbfunc, cbdata); } pmix_status_t dmodex_fn(const pmix_proc_t *proc, @@ -153,12 +142,12 @@ pmix_status_t dmodex_fn(const pmix_proc_t *proc, { TEST_VERBOSE(("Getting data for %s:%d", proc->nspace, proc->rank)); - /* In a perfect world, we should call another server - * to get the data for one of its clients. We don't - * have multi-server capability yet, so we'll just - * respond right away */ - - return PMIX_ERR_NOT_FOUND; + /* return not_found fot single server mode */ + if ((pmix_list_get_size(server_list) == 1) && (my_server_id == 0)) { + return PMIX_ERR_NOT_FOUND; + } + // TODO: add support tracker for dmodex requests + return server_dmdx_get(proc->nspace, proc->rank, cbfunc, cbdata); } pmix_status_t publish_fn(const pmix_proc_t *proc, @@ -199,6 +188,7 @@ pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, pmix_lookup_cbfunc_t cbfunc, void *cbdata) { size_t i, ndata, ret; + pmix_status_t rc = PMIX_SUCCESS; pmix_pdata_t *pdata; pmix_test_info_t *tinfo; if (NULL == pmix_test_published_list) { @@ -221,13 +211,15 @@ pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, } } if (ret != ndata) { - return PMIX_ERR_NOT_FOUND; + rc = PMIX_ERR_NOT_FOUND; + goto error; } if (NULL != cbfunc) { cbfunc(PMIX_SUCCESS, pdata, ndata, cbdata); } +error: PMIX_PDATA_FREE(pdata, ndata); - return PMIX_SUCCESS; + return rc; } pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys, diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simpclient.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simpclient.c index ca277c5e28..ae5b4ababc 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simpclient.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simpclient.c @@ -184,6 +184,7 @@ int main(int argc, char **argv) usleep(10); } + /* get our universe size */ (void)strncpy(proc.nspace, myproc.nspace, PMIX_MAX_NSLEN); proc.rank = PMIX_RANK_WILDCARD; diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simpdie.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simpdie.c index db62d7832c..fd6a61eba4 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simpdie.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simpdie.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -87,7 +87,9 @@ int main(int argc, char **argv) pmix_value_t *val = &value; pmix_proc_t proc; uint32_t nprocs; - + pmix_status_t code[5] = {PMIX_ERR_PROC_ABORTING, PMIX_ERR_PROC_ABORTED, + PMIX_ERR_PROC_REQUESTED_ABORT, PMIX_ERR_JOB_TERMINATED, + PMIX_ERR_UNREACH}; /* init us */ if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) { pmix_output(0, "Client ns %s rank %d: PMIx_Init failed: %d", myproc.nspace, myproc.rank, rc); @@ -108,7 +110,7 @@ int main(int argc, char **argv) completed = false; /* register our errhandler */ - PMIx_Register_event_handler(NULL, 0, NULL, 0, + PMIx_Register_event_handler(code, 5, NULL, 0, notification_fn, errhandler_reg_callbk, NULL); /* call fence to sync */ diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simpdmodex.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simpdmodex.c index b12afdb9b3..c042f1948a 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simpdmodex.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simpdmodex.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -144,7 +144,7 @@ int main(int argc, char **argv) (void)asprintf(&tmp, "%s-%d-remote", myproc.nspace, myproc.rank); value.type = PMIX_STRING; value.data.string = "1234"; - if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_REMOTE, tmp, &value))) { + if (PMIX_SUCCESS != (rc = PMIx_Put(PMIX_GLOBAL, tmp, &value))) { pmix_output(0, "Client ns %s rank %d: PMIx_Put internal failed: %d", myproc.nspace, myproc.rank, rc); goto done; } diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c index fa7d3fb933..e31f4211b9 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simptest.c @@ -14,8 +14,8 @@ * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -343,7 +343,6 @@ int main(int argc, char **argv) pmix_info_t *info; size_t ninfo; bool cross_version = false; - bool usock = true; bool hwloc = false; #if PMIX_HAVE_HWLOC char *hwloc_file = NULL; @@ -357,8 +356,6 @@ int main(int argc, char **argv) exit(1); } - fprintf(stderr, "Testing version %s\n", PMIx_Get_version()); - /* see if we were passed the number of procs to run or * the executable to use */ for (n=1; n < argc; n++) { @@ -381,25 +378,6 @@ int main(int argc, char **argv) /* cross-version test - we will set one child to * run at a different version. Requires -n >= 2 */ cross_version = true; - usock = false; - } else if (0 == strcmp("-u", argv[n])) { - /* enable usock */ - usock = false; -#if PMIX_HAVE_HWLOC - } else if (0 == strcmp("-hwloc", argv[n]) || - 0 == strcmp("--hwloc", argv[n])) { - /* test hwloc support */ - hwloc = true; - } else if (0 == strcmp("-hwloc-file", argv[n]) || - 0 == strcmp("--hwloc-file", argv[n])) { - if (NULL == argv[n+1]) { - fprintf(stderr, "The --hwloc-file option requires an argument\n"); - exit(1); - } - hwloc_file = strdup(argv[n+1]); - hwloc = true; - ++n; -#endif } else if (0 == strcmp("-h", argv[n])) { /* print the options and exit */ fprintf(stderr, "usage: simptest \n"); @@ -427,34 +405,35 @@ int main(int argc, char **argv) } #endif + fprintf(stderr, "Testing version %s\n", PMIx_Get_version()); + /* setup the server library and tell it to support tool connections */ #if PMIX_HAVE_HWLOC if (hwloc) { #if HWLOC_API_VERSION < 0x20000 - ninfo = 4; + ninfo = 3; #else - ninfo = 5; + ninfo = 4; #endif } else { - ninfo = 3; + ninfo = 2; } #else - ninfo = 3; + ninfo = 2; #endif PMIX_INFO_CREATE(info, ninfo); PMIX_INFO_LOAD(&info[0], PMIX_SERVER_TOOL_SUPPORT, NULL, PMIX_BOOL); - PMIX_INFO_LOAD(&info[1], PMIX_USOCK_DISABLE, &usock, PMIX_BOOL); - PMIX_INFO_LOAD(&info[2], PMIX_SERVER_GATEWAY, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&info[1], PMIX_SERVER_GATEWAY, NULL, PMIX_BOOL); #if PMIX_HAVE_HWLOC if (hwloc) { if (NULL != hwloc_file) { - PMIX_INFO_LOAD(&info[3], PMIX_TOPOLOGY_FILE, hwloc_file, PMIX_STRING); + PMIX_INFO_LOAD(&info[2], PMIX_TOPOLOGY_FILE, hwloc_file, PMIX_STRING); } else { - PMIX_INFO_LOAD(&info[3], PMIX_TOPOLOGY, NULL, PMIX_STRING); + PMIX_INFO_LOAD(&info[2], PMIX_TOPOLOGY, NULL, PMIX_STRING); } #if HWLOC_API_VERSION >= 0x20000 - PMIX_INFO_LOAD(&info[4], PMIX_HWLOC_SHARE_TOPO, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&info[3], PMIX_HWLOC_SHARE_TOPO, NULL, PMIX_BOOL); #endif } #endif @@ -560,9 +539,6 @@ int main(int argc, char **argv) } else { pmix_setenv("PMIX_MCA_ptl", "usock", true, &client_env); } - } else if (!usock) { - /* don't disable usock => enable it on client */ - pmix_setenv("PMIX_MCA_ptl", "usock", true, &client_env); } x = PMIX_NEW(myxfer_t); if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, @@ -599,7 +575,6 @@ int main(int argc, char **argv) pmix_list_append(&children, &child->super); } } - free(executable); pmix_argv_free(client_argv); pmix_argv_free(client_env); @@ -611,14 +586,21 @@ int main(int argc, char **argv) nanosleep(&ts, NULL); } - /* see if anyone exited with non-zero status */ - n=0; - PMIX_LIST_FOREACH(child, &children, wait_tracker_t) { - if (0 != child->exit_code) { - fprintf(stderr, "Child %d [%d] exited with status %d - test FAILED\n", n, child->pid, child->exit_code); - } - ++n; + /* see if anyone exited with non-zero status unless the test + * was expected to do so */ + if (NULL == strstr(executable, "simpdie")) { + n=0; + PMIX_LIST_FOREACH(child, &children, wait_tracker_t) { + if (0 != child->exit_code) { + fprintf(stderr, "Child %d [%d] exited with status %d - test FAILED\n", n, child->pid, child->exit_code); + } + ++n; + } + } else if (1 == exit_code) { + exit_code = 0; } + free(executable); + /* try notifying ourselves */ ninfo = 3; PMIX_INFO_CREATE(info, ninfo); @@ -729,21 +711,14 @@ static void errhandler_reg_callbk (pmix_status_t status, static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_output(0, "SERVER: FINALIZED %s:%d WAKEUP %d", proc->nspace, proc->rank, wakeup); - /* ensure we call the cbfunc so the proc can exit! */ - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static void abcbfunc(pmix_status_t status, void *cbdata) @@ -801,17 +776,31 @@ static pmix_status_t abort_fn(const pmix_proc_t *proc, return PMIX_SUCCESS; } +static void fencbfn(int sd, short args, void *cbdata) +{ + pmix_shift_caddy_t *scd = (pmix_shift_caddy_t*)cbdata; + /* pass the provided data back to each participating proc */ + if (NULL != scd->cbfunc.modexcbfunc) { + scd->cbfunc.modexcbfunc(scd->status, scd->data, scd->ndata, scd->cbdata, NULL, NULL); + } + PMIX_RELEASE(scd); +} static pmix_status_t fencenb_fn(const pmix_proc_t procs[], size_t nprocs, const pmix_info_t info[], size_t ninfo, char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata) { + pmix_shift_caddy_t *scd; + pmix_output(0, "SERVER: FENCENB"); - /* pass the provided data back to each participating proc */ - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, data, ndata, cbdata, NULL, NULL); - } + scd = PMIX_NEW(pmix_shift_caddy_t); + scd->status = PMIX_SUCCESS; + scd->data = data; + scd->ndata = ndata; + scd->cbfunc.modexcbfunc = cbfunc; + scd->cbdata = cbdata; + PMIX_THREADSHIFT(scd, fencbfn); return PMIX_SUCCESS; } @@ -820,6 +809,8 @@ static pmix_status_t dmodex_fn(const pmix_proc_t *proc, const pmix_info_t info[], size_t ninfo, pmix_modex_cbfunc_t cbfunc, void *cbdata) { + pmix_shift_caddy_t *scd; + pmix_output(0, "SERVER: DMODEX"); /* if this is a timeout test, then do nothing */ @@ -827,11 +818,12 @@ static pmix_status_t dmodex_fn(const pmix_proc_t *proc, return PMIX_SUCCESS; } - /* we don't have any data for remote procs as this - * test only runs one server - so report accordingly */ - if (NULL != cbfunc) { - cbfunc(PMIX_ERR_NOT_FOUND, NULL, 0, cbdata, NULL, NULL); - } + scd = PMIX_NEW(pmix_shift_caddy_t); + scd->status = PMIX_ERR_NOT_FOUND; + scd->cbfunc.modexcbfunc = cbfunc; + scd->cbdata = cbdata; + PMIX_THREADSHIFT(scd, fencbfn); + return PMIX_SUCCESS; } @@ -853,12 +845,26 @@ static pmix_status_t publish_fn(const pmix_proc_t *proc, pmix_value_xfer(&p->pdata.value, (pmix_value_t*)&info[n].value); pmix_list_append(&pubdata, &p->super); } - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + + return PMIX_OPERATION_SUCCEEDED; } +typedef struct { + pmix_event_t ev; + pmix_pdata_t *pd; + size_t n; + pmix_lookup_cbfunc_t cbfunc; + void *cbdata; +} lkobj_t; + +static void lkcbfn(int sd, short args, void *cbdata) +{ + lkobj_t *lk = (lkobj_t*)cbdata; + + lk->cbfunc(PMIX_SUCCESS, lk->pd, lk->n, lk->cbdata); + PMIX_PDATA_FREE(lk->pd, lk->n); + free(lk); +} static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, const pmix_info_t info[], size_t ninfo, @@ -869,6 +875,7 @@ static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, size_t i, n; pmix_pdata_t *pd = NULL; pmix_status_t ret = PMIX_ERR_NOT_FOUND; + lkobj_t *lk; pmix_output(0, "SERVER: LOOKUP"); @@ -901,13 +908,16 @@ static pmix_status_t lookup_fn(const pmix_proc_t *proc, char **keys, } } PMIX_LIST_DESTRUCT(&results); - if (NULL != cbfunc) { - cbfunc(ret, pd, n, cbdata); + if (PMIX_SUCCESS == ret) { + lk = (lkobj_t*)malloc(sizeof(lkobj_t)); + lk->pd = pd; + lk->n = n; + lk->cbfunc = cbfunc; + lk->cbdata = cbdata; + PMIX_THREADSHIFT(lk, lkcbfn); } - if (0 < n) { - PMIX_PDATA_FREE(pd, n); - } - return PMIX_SUCCESS; + + return ret; } @@ -929,10 +939,7 @@ static pmix_status_t unpublish_fn(const pmix_proc_t *proc, char **keys, } } } - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static void spcbfunc(pmix_status_t status, void *cbdata) @@ -996,11 +1003,7 @@ static pmix_status_t connect_fn(const pmix_proc_t procs[], size_t nprocs, numconnects++; - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } @@ -1010,30 +1013,20 @@ static pmix_status_t disconnect_fn(const pmix_proc_t procs[], size_t nprocs, { pmix_output(0, "SERVER: DISCONNECT"); - /* in practice, we would pass this request to the local - * resource manager for handling */ - - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t register_event_fn(pmix_status_t *codes, size_t ncodes, const pmix_info_t info[], size_t ninfo, pmix_op_cbfunc_t cbfunc, void *cbdata) { - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t deregister_events(pmix_status_t *codes, size_t ncodes, pmix_op_cbfunc_t cbfunc, void *cbdata) { - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t notify_event(pmix_status_t code, @@ -1047,10 +1040,21 @@ static pmix_status_t notify_event(pmix_status_t code, } typedef struct query_data_t { + pmix_event_t ev; pmix_info_t *data; size_t ndata; + pmix_info_cbfunc_t cbfunc; + void *cbdata; } query_data_t; +static void qfn(int sd, short args, void *cbdata) +{ + query_data_t *qd = (query_data_t*)cbdata; + + qd->cbfunc(PMIX_SUCCESS, qd->data, qd->ndata, qd->cbdata, NULL, NULL); + PMIX_INFO_FREE(qd->data, qd->ndata); +} + static pmix_status_t query_fn(pmix_proc_t *proct, pmix_query_t *queries, size_t nqueries, pmix_info_cbfunc_t cbfunc, @@ -1058,6 +1062,7 @@ static pmix_status_t query_fn(pmix_proc_t *proct, { size_t n; pmix_info_t *info; + query_data_t qd; pmix_output(0, "SERVER: QUERY"); @@ -1074,7 +1079,11 @@ static pmix_status_t query_fn(pmix_proc_t *proct, return PMIX_ERROR; } } - cbfunc(PMIX_SUCCESS, info, nqueries, cbdata, NULL, NULL); + qd.data = info; + qd.ndata = nqueries; + qd.cbfunc = cbfunc; + qd.cbdata = cbdata; + PMIX_THREADSHIFT(&qd, qfn); return PMIX_SUCCESS; } @@ -1095,16 +1104,29 @@ static void tool_connect_fn(pmix_info_t *info, size_t ninfo, } } +typedef struct { + pmix_event_t ev; + pmix_op_cbfunc_t cbfunc; + void *cbdata; +} mylog_t; + +static void foobar(int sd, short args, void *cbdata) +{ + mylog_t *lg = (mylog_t*)cbdata; + lg->cbfunc(PMIX_SUCCESS, lg->cbdata); +} static void log_fn(const pmix_proc_t *client, const pmix_info_t data[], size_t ndata, const pmix_info_t directives[], size_t ndirs, pmix_op_cbfunc_t cbfunc, void *cbdata) { + mylog_t *lg = (mylog_t *)malloc(sizeof(mylog_t)); + pmix_output(0, "SERVER: LOG"); - if (NULL != cbfunc) { - cbfunc(PMIX_SUCCESS, cbdata); - } + lg->cbfunc = cbfunc; + lg->cbdata = cbdata; + PMIX_THREADSHIFT(lg, foobar); } static pmix_status_t alloc_fn(const pmix_proc_t *client, @@ -1112,7 +1134,7 @@ static pmix_status_t alloc_fn(const pmix_proc_t *client, const pmix_info_t data[], size_t ndata, pmix_info_cbfunc_t cbfunc, void *cbdata) { - return PMIX_SUCCESS; + return PMIX_OPERATION_SUCCEEDED; } static pmix_status_t jctrl_fn(const pmix_proc_t *requestor, @@ -1160,13 +1182,19 @@ static void wait_signal_callback(int fd, short event, void *arg) /* we are already in an event, so it is safe to access the list */ PMIX_LIST_FOREACH(t2, &children, wait_tracker_t) { if (pid == t2->pid) { - t2->exit_code = status; /* found it! */ - if (0 != status && 0 == exit_code) { - exit_code = status; + if (WIFEXITED(status)) { + t2->exit_code = WEXITSTATUS(status); + } else { + if (WIFSIGNALED(status)) { + t2->exit_code = WTERMSIG(status) + 128; + } + } + if (0 != t2->exit_code && 0 == exit_code) { + exit_code = t2->exit_code; } --wakeup; - return; + break; } } } diff --git a/opal/mca/pmix/pmix3x/pmix/test/simple/simptool.c b/opal/mca/pmix/pmix3x/pmix/test/simple/simptool.c index 2af6f395ed..9e96d21ee7 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/simple/simptool.c +++ b/opal/mca/pmix/pmix3x/pmix/test/simple/simptool.c @@ -13,7 +13,7 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * @@ -95,6 +95,7 @@ int main(int argc, char **argv) PMIX_QUERY_CREATE(query, nq); pmix_argv_append_nosize(&query[0].keys, "foobar"); pmix_argv_append_nosize(&query[1].keys, "spastic"); + pmix_argv_append_nosize(&query[1].keys, PMIX_SERVER_URI); active = true; if (PMIX_SUCCESS != (rc = PMIx_Query_info_nb(query, nq, cbfunc, (void*)&active))) { pmix_output(0, "Client ns %s rank %d: PMIx_Query_info failed: %d", myproc.nspace, myproc.rank, rc); diff --git a/opal/mca/pmix/pmix3x/pmix/test/test_common.c b/opal/mca/pmix/pmix3x/pmix/test/test_common.c index 9021e58ba2..7b9ac8701d 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/test_common.c +++ b/opal/mca/pmix/pmix3x/pmix/test/test_common.c @@ -1,8 +1,8 @@ /* - * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. - * Copyright (c) 2015-2017 Mellanox Technologies, Inc. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -87,6 +87,11 @@ void parse_cmd(int argc, char **argv, test_params *params) if (NULL != argv[i]) { params->binary = strdup(argv[i]); } + } else if (0 == strcmp(argv[i], "--nservers") || 0 == strcmp(argv[i], "-s")){ + i++; + if (NULL != argv[i]) { + params->nservers = atoi(argv[i]); + } } else if( 0 == strcmp(argv[i], "--verbose") || 0 == strcmp(argv[i],"-v") ){ TEST_VERBOSE_ON(); params->verbose = 1; @@ -600,7 +605,6 @@ int get_total_ns_number(test_params params) int get_all_ranks_from_namespace(test_params params, char *nspace, pmix_proc_t **ranks, size_t *nranks) { - int base_rank = 0; size_t num_ranks = 0; int num = -1; size_t j; @@ -616,7 +620,6 @@ int get_all_ranks_from_namespace(test_params params, char *nspace, pmix_proc_t * char *pch = tmp; int ns_id = (int)strtol(nspace + strlen(TEST_NAMESPACE) + 1, NULL, 10); while (NULL != pch && num != ns_id) { - base_rank += num_ranks; pch = strtok((-1 == num ) ? tmp : NULL, ":"); if (NULL == pch) { break; @@ -629,7 +632,7 @@ int get_all_ranks_from_namespace(test_params params, char *nspace, pmix_proc_t * PMIX_PROC_CREATE(*ranks, num_ranks); for (j = 0; j < num_ranks; j++) { (void)strncpy((*ranks)[j].nspace, nspace, PMIX_MAX_NSLEN); - (*ranks)[j].rank = base_rank+j; + (*ranks)[j].rank = j; } } else { free(tmp); diff --git a/opal/mca/pmix/pmix3x/pmix/test/test_common.h b/opal/mca/pmix/pmix3x/pmix/test/test_common.h index acc49d1bce..fd25f8bdf8 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/test_common.h +++ b/opal/mca/pmix/pmix3x/pmix/test/test_common.h @@ -1,10 +1,10 @@ /* - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015-2017 Mellanox Technologies, Inc. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -51,7 +51,7 @@ extern FILE *file; #define STRIPPED_FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) #define TEST_OUTPUT(x) { \ - fprintf(file,"%s:%s: %s\n",STRIPPED_FILE_NAME, __func__, \ + fprintf(file,"==%d== %s:%s: %s\n", getpid(), STRIPPED_FILE_NAME, __func__, \ pmix_test_output_prepare x ); \ fflush(file); \ } @@ -59,13 +59,13 @@ extern FILE *file; // Write output without adding anything to it. // Need for automate tests to receive "OK" string #define TEST_OUTPUT_CLEAR(x) { \ - fprintf(file, "%s", pmix_test_output_prepare x ); \ + fprintf(file, "==%d== %s", getpid(), pmix_test_output_prepare x ); \ fflush(file); \ } // Always write errors to the stderr #define TEST_ERROR(x) { \ - fprintf(stderr,"ERROR [%s:%d:%s]: %s\n", STRIPPED_FILE_NAME, __LINE__, __func__, \ + fprintf(stderr,"==%d== ERROR [%s:%d:%s]: %s\n", getpid(), STRIPPED_FILE_NAME, __LINE__, __func__, \ pmix_test_output_prepare x ); \ fflush(stderr); \ } @@ -129,6 +129,8 @@ typedef struct { char *key_replace; int test_internal; char *gds_mode; + int nservers; + uint32_t lsize; } test_params; #define INIT_TEST_PARAMS(params) do { \ @@ -160,6 +162,8 @@ typedef struct { params.key_replace = NULL; \ params.test_internal = 0; \ params.gds_mode = NULL; \ + params.nservers = 1; \ + params.lsize = 0; \ } while (0) #define FREE_TEST_PARAMS(params) do { \ diff --git a/opal/mca/pmix/pmix3x/pmix/test/test_server.c b/opal/mca/pmix/pmix3x/pmix/test/test_server.c new file mode 100644 index 0000000000..3627dade91 --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/test/test_server.c @@ -0,0 +1,978 @@ + /* + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include +#include +#include +#include +#include + +#include "pmix_server.h" +#include "src/include/pmix_globals.h" + +#include "test_server.h" +#include "test_common.h" +#include "cli_stages.h" +#include "server_callbacks.h" + +int my_server_id = 0; + +server_info_t *my_server_info = NULL; +pmix_list_t *server_list = NULL; +pmix_list_t *server_nspace = NULL; + +static void sdes(server_info_t *s) +{ + close(s->rd_fd); + close(s->wr_fd); + if (s->evread) { + event_del(s->evread); + } + s->evread = NULL; +} + +static void scon(server_info_t *s) +{ + s->idx = 0; + s->pid = 0; + s->rd_fd = -1; + s->wr_fd = -1; + s->evread = NULL; + s->modex_cbfunc = NULL; + s->cbdata = NULL; +} + +PMIX_CLASS_INSTANCE(server_info_t, + pmix_list_item_t, + scon, sdes); + +static void nsdes(server_nspace_t *ns) +{ + if (ns->task_map) { + free(ns->task_map); + } +} + +static void nscon(server_nspace_t *ns) +{ + memset(ns->name, 0, PMIX_MAX_NSLEN); + ns->ntasks = 0; + ns->task_map = NULL; +} + +PMIX_CLASS_INSTANCE(server_nspace_t, + pmix_list_item_t, + nscon, nsdes); + +static int server_send_procs(void); +static void server_read_cb(evutil_socket_t fd, short event, void *arg); +static int srv_wait_all(double timeout); +static int server_fwd_msg(msg_hdr_t *msg_hdr, char *buf, size_t size); +static int server_send_msg(msg_hdr_t *msg_hdr, char *data, size_t size); +static void remove_server_item(server_info_t *server); +static void server_unpack_dmdx(char *buf, int *sender, pmix_proc_t *proc); +static int server_pack_dmdx(int sender_id, const char *nspace, int rank, + char **buf); +static void _dmdx_cb(int status, char *data, size_t sz, void *cbdata); + +static void release_cb(pmix_status_t status, void *cbdata) +{ + int *ptr = (int*)cbdata; + *ptr = 0; +} + +static void set_namespace(int local_size, int univ_size, char *ranks, char *name) +{ + size_t ninfo; + pmix_info_t *info; + ninfo = 8; + char *regex, *ppn; + + PMIX_INFO_CREATE(info, ninfo); + pmix_strncpy(info[0].key, PMIX_UNIV_SIZE, PMIX_MAX_KEYLEN); + info[0].value.type = PMIX_UINT32; + info[0].value.data.uint32 = univ_size; + + pmix_strncpy(info[1].key, PMIX_SPAWNED, PMIX_MAX_KEYLEN); + info[1].value.type = PMIX_UINT32; + info[1].value.data.uint32 = 0; + + pmix_strncpy(info[2].key, PMIX_LOCAL_SIZE, PMIX_MAX_KEYLEN); + info[2].value.type = PMIX_UINT32; + info[2].value.data.uint32 = local_size; + + pmix_strncpy(info[3].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN); + info[3].value.type = PMIX_STRING; + info[3].value.data.string = strdup(ranks); + + PMIx_generate_regex(NODE_NAME, ®ex); + pmix_strncpy(info[4].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN); + info[4].value.type = PMIX_STRING; + info[4].value.data.string = regex; + + PMIx_generate_ppn(ranks, &ppn); + pmix_strncpy(info[5].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN); + info[5].value.type = PMIX_STRING; + info[5].value.data.string = ppn; + + pmix_strncpy(info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN); + info[6].value.type = PMIX_UINT32; + info[6].value.data.uint32 = univ_size; + + pmix_strncpy(info[7].key, PMIX_APPNUM, PMIX_MAX_KEYLEN); + info[7].value.type = PMIX_UINT32; + info[7].value.data.uint32 = getpid (); + + int in_progress = 1, rc; + if (PMIX_SUCCESS == (rc = PMIx_server_register_nspace(name, local_size, + info, ninfo, release_cb, &in_progress))) { + PMIX_WAIT_FOR_COMPLETION(in_progress); + } + PMIX_INFO_FREE(info, ninfo); +} + +static void fill_seq_ranks_array(size_t nprocs, int base_rank, char **ranks) +{ + uint32_t i; + int len = 0, max_ranks_len; + if (0 >= nprocs) { + return; + } + max_ranks_len = nprocs * (MAX_DIGIT_LEN+1); + *ranks = (char*) malloc(max_ranks_len); + for (i = 0; i < nprocs; i++) { + len += snprintf(*ranks + len, max_ranks_len-len-1, "%d", i+base_rank); + if (i != nprocs-1) { + len += snprintf(*ranks + len, max_ranks_len-len-1, "%c", ','); + } + } + if (len >= max_ranks_len-1) { + free(*ranks); + *ranks = NULL; + TEST_ERROR(("Not enough allocated space for global ranks array.")); + } +} + +static void server_unpack_procs(char *buf, size_t size) +{ + char *ptr = buf; + size_t i; + size_t ns_count; + char *nspace; + + while ((size_t)(ptr - buf) < size) { + ns_count = (size_t)*ptr; + ptr += sizeof(size_t); + + for (i = 0; i < ns_count; i++) { + server_nspace_t *tmp, *ns_item = NULL; + size_t ltasks, ntasks; + int server_id; + + server_id = *ptr; + ptr += sizeof(int); + + nspace = ptr; + ptr += PMIX_MAX_NSLEN+1; + + ntasks = (size_t)*ptr; + ptr += sizeof(size_t); + + ltasks = (size_t)*ptr; + ptr += sizeof(size_t); + + PMIX_LIST_FOREACH(tmp, server_nspace, server_nspace_t) { + if (0 == strcmp(nspace, tmp->name)) { + ns_item = tmp; + break; + } + } + if (NULL == ns_item) { + ns_item = PMIX_NEW(server_nspace_t); + memcpy(ns_item->name, nspace, PMIX_MAX_NSLEN); + pmix_list_append(server_nspace, &ns_item->super); + ns_item->ltasks = ltasks; + ns_item->ntasks = ntasks; + ns_item->task_map = (int*)malloc(sizeof(int) * ntasks); + memset(ns_item->task_map, -1, sizeof(int) * ntasks); + } else { + assert(ns_item->ntasks == ntasks); + } + size_t i; + for (i = 0; i < ltasks; i++) { + int rank = (int)*ptr; + ptr += sizeof(int); + if (ns_item->task_map[rank] >= 0) { + continue; + } + ns_item->task_map[rank] = server_id; + } + } + } +} + +static size_t server_pack_procs(int server_id, char **buf, size_t size) +{ + size_t ns_count = pmix_list_get_size(server_nspace); + size_t buf_size = sizeof(size_t) + (PMIX_MAX_NSLEN+1)*ns_count; + server_nspace_t *tmp; + char *ptr; + + if (0 == ns_count) { + return 0; + } + + buf_size += size; + /* compute size: server_id + total + local procs count + ranks */ + PMIX_LIST_FOREACH(tmp, server_nspace, server_nspace_t) { + buf_size += sizeof(int) + sizeof(size_t) + sizeof(size_t) + + sizeof(int) * tmp->ltasks; + } + *buf = (char*)realloc(*buf, buf_size); + memset(*buf + size, 0, buf_size); + ptr = *buf + size; + /* pack ns count */ + memcpy(ptr, &ns_count, sizeof(size_t)); + ptr += sizeof(size_t); + + assert(server_nspace->pmix_list_length); + + PMIX_LIST_FOREACH(tmp, server_nspace, server_nspace_t) { + size_t i; + /* pack server_id */ + memcpy(ptr, &server_id, sizeof(int)); + ptr += sizeof(int); + /* pack ns name */ + memcpy(ptr, tmp->name, PMIX_MAX_NSLEN+1); + ptr += PMIX_MAX_NSLEN+1; + /* pack ns total size */ + memcpy(ptr, &tmp->ntasks, sizeof(size_t)); + ptr += sizeof(size_t); + /* pack ns local size */ + memcpy(ptr, &tmp->ltasks, sizeof(size_t)); + ptr += sizeof(size_t); + /* pack ns ranks */ + for(i = 0; i < tmp->ntasks; i++) { + if (tmp->task_map[i] == server_id) { + int rank = (int)i; + memcpy(ptr, &rank, sizeof(int)); + ptr += sizeof(int); + } + } + } + assert((size_t)(ptr - *buf) == buf_size); + return buf_size; +} + +static void remove_server_item(server_info_t *server) +{ + pmix_list_remove_item(server_list, &server->super); + PMIX_DESTRUCT_LOCK(&server->lock); + PMIX_RELEASE(server); +} + +static int srv_wait_all(double timeout) +{ + server_info_t *server, *next; + pid_t pid; + int status; + struct timeval tv; + double start_time, cur_time; + int ret = 0; + + gettimeofday(&tv, NULL); + start_time = tv.tv_sec + 1E-6*tv.tv_usec; + cur_time = start_time; + + /* Remove this server from the list */ + PMIX_LIST_FOREACH_SAFE(server, next, server_list, server_info_t) { + if (server->pid == getpid()) { + /* remove himself */ + remove_server_item(server); + break; + } + } + + while (!pmix_list_is_empty(server_list) && + (timeout >= (cur_time - start_time))) { + pid = waitpid(-1, &status, 0); + if (pid >= 0) { + PMIX_LIST_FOREACH_SAFE(server, next, server_list, server_info_t) { + if (server->pid == pid) { + TEST_VERBOSE(("server %d finalize PID:%d with status %d", server->idx, + server->pid, WEXITSTATUS(status))); + ret += WEXITSTATUS(status); + remove_server_item(server); + } + } + } + // calculate current timestamp + gettimeofday(&tv, NULL); + cur_time = tv.tv_sec + 1E-6*tv.tv_usec; + } + + return ret; +} + +static int server_fwd_msg(msg_hdr_t *msg_hdr, char *buf, size_t size) +{ + server_info_t *tmp_server, *server = NULL; + int rc = PMIX_SUCCESS; + + PMIX_LIST_FOREACH(tmp_server, server_list, server_info_t) { + if (tmp_server->idx == msg_hdr->dst_id) { + server = tmp_server; + break; + } + } + if (NULL == server) { + return PMIX_ERROR; + } + rc = write(server->wr_fd, msg_hdr, sizeof(msg_hdr_t)); + if (rc != sizeof(msg_hdr_t)) { + return PMIX_ERROR; + } + rc = write(server->wr_fd, buf, size); + if (rc != (ssize_t)size) { + return PMIX_ERROR; + } + return PMIX_SUCCESS; +} + +static int server_send_msg(msg_hdr_t *msg_hdr, char *data, size_t size) +{ + size_t ret = 0; + server_info_t *server = NULL, *server_tmp; + if (0 == my_server_id) { + PMIX_LIST_FOREACH(server_tmp, server_list, server_info_t) { + if (server_tmp->idx == msg_hdr->dst_id) { + server = server_tmp; + break; + } + } + if (NULL == server) { + abort(); + } + } else { + server = (server_info_t *)pmix_list_get_first(server_list); + } + + ret += write(server->wr_fd, msg_hdr, sizeof(msg_hdr_t)); + ret += write(server->wr_fd, data, size); + if (ret != (sizeof(*msg_hdr) + size)) { + return PMIX_ERROR; + } + return PMIX_SUCCESS; +} + +static void _send_procs_cb(pmix_status_t status, const char *data, + size_t ndata, void *cbdata, + pmix_release_cbfunc_t relfn, void *relcbd) +{ + server_info_t *server = (server_info_t*)cbdata; + + server_unpack_procs((char*)data, ndata); + free((char*)data); + PMIX_WAKEUP_THREAD(&server->lock); +} + +static int server_send_procs(void) +{ + server_info_t *server; + msg_hdr_t msg_hdr; + int rc = PMIX_SUCCESS; + char *buf = NULL; + + if (0 == my_server_id) { + server = my_server_info; + } else { + server = (server_info_t *)pmix_list_get_first(server_list); + } + + msg_hdr.cmd = CMD_FENCE_CONTRIB; + msg_hdr.dst_id = 0; + msg_hdr.src_id = my_server_id; + msg_hdr.size = server_pack_procs(my_server_id, &buf, 0); + server->modex_cbfunc = _send_procs_cb; + server->cbdata = (void*)server; + + server->lock.active = true; + + if (PMIX_SUCCESS != (rc = server_send_msg(&msg_hdr, buf, msg_hdr.size))) { + if (buf) { + free(buf); + } + return PMIX_ERROR; + } + if (buf) { + free(buf); + } + + PMIX_WAIT_THREAD(&server->lock); + return PMIX_SUCCESS; +} + +int server_barrier(void) +{ + server_info_t *server; + msg_hdr_t msg_hdr; + int rc = PMIX_SUCCESS; + + if (0 == my_server_id) { + server = my_server_info; + } else { + server = (server_info_t *)pmix_list_get_first(server_list); + } + + msg_hdr.cmd = CMD_BARRIER_REQUEST; + msg_hdr.dst_id = 0; + msg_hdr.src_id = my_server_id; + msg_hdr.size = 0; + + server->lock.active = true; + + if (PMIX_SUCCESS != (rc = server_send_msg(&msg_hdr, NULL, 0))) { + return PMIX_ERROR; + } + PMIX_WAIT_THREAD(&server->lock); + + return PMIX_SUCCESS; +} + +static void _libpmix_cb(void *cbdata) +{ + char *ptr = (char*)cbdata; + if (ptr) { + free(ptr); + } +} + +static void server_read_cb(evutil_socket_t fd, short event, void *arg) +{ + server_info_t *server = (server_info_t*)arg; + msg_hdr_t msg_hdr; + char *msg_buf = NULL; + static char *fence_buf = NULL; + int rc; + static size_t barrier_cnt = 0; + static size_t contrib_cnt = 0; + static size_t fence_buf_offset = 0; + + rc = read(server->rd_fd, &msg_hdr, sizeof(msg_hdr_t)); + if (rc <= 0) { + return; + } + if (msg_hdr.size) { + msg_buf = (char*) malloc(sizeof(char) * msg_hdr.size); + rc += read(server->rd_fd, msg_buf, msg_hdr.size); + } + if (rc != (int)(sizeof(msg_hdr_t) + msg_hdr.size)) { + TEST_ERROR(("error read from %d", server->idx)); + } + + if (my_server_id != msg_hdr.dst_id) { + server_fwd_msg(&msg_hdr, msg_buf, msg_hdr.size); + free(msg_buf); + return; + } + + switch(msg_hdr.cmd) { + case CMD_BARRIER_REQUEST: + barrier_cnt++; + TEST_VERBOSE(("CMD_BARRIER_REQ req from %d cnt %d", msg_hdr.src_id, + barrier_cnt)); + if (pmix_list_get_size(server_list) == barrier_cnt) { + barrier_cnt = 0; /* reset barrier counter */ + server_info_t *tmp_server; + PMIX_LIST_FOREACH(tmp_server, server_list, server_info_t) { + msg_hdr_t resp_hdr; + resp_hdr.dst_id = tmp_server->idx; + resp_hdr.src_id = my_server_id; + resp_hdr.cmd = CMD_BARRIER_RESPONSE; + resp_hdr.size = 0; + server_send_msg(&resp_hdr, NULL, 0); + } + } + break; + case CMD_BARRIER_RESPONSE: + TEST_VERBOSE(("%d: CMD_BARRIER_RESP", my_server_id)); + PMIX_WAKEUP_THREAD(&server->lock); + break; + case CMD_FENCE_CONTRIB: + contrib_cnt++; + if (msg_hdr.size > 0) { + fence_buf = (char*)realloc((void*)fence_buf, + fence_buf_offset + msg_hdr.size); + memcpy(fence_buf + fence_buf_offset, msg_buf, msg_hdr.size); + fence_buf_offset += msg_hdr.size; + free(msg_buf); + msg_buf = NULL; + } + + TEST_VERBOSE(("CMD_FENCE_CONTRIB req from %d cnt %d size %d", + msg_hdr.src_id, contrib_cnt, msg_hdr.size)); + if (pmix_list_get_size(server_list) == contrib_cnt) { + server_info_t *tmp_server; + PMIX_LIST_FOREACH(tmp_server, server_list, server_info_t) { + msg_hdr_t resp_hdr; + resp_hdr.dst_id = tmp_server->idx; + resp_hdr.src_id = my_server_id; + resp_hdr.cmd = CMD_FENCE_COMPLETE; + resp_hdr.size = fence_buf_offset; + server_send_msg(&resp_hdr, fence_buf, fence_buf_offset); + } + TEST_VERBOSE(("CMD_FENCE_CONTRIB complete, size %d", + fence_buf_offset)); + if (fence_buf) { + free(fence_buf); + fence_buf = NULL; + fence_buf_offset = 0; + } + contrib_cnt = 0; + } + break; + case CMD_FENCE_COMPLETE: + TEST_VERBOSE(("%d: CMD_FENCE_COMPLETE size %d", my_server_id, + msg_hdr.size)); + server->modex_cbfunc(PMIX_SUCCESS, msg_buf, msg_hdr.size, + server->cbdata, _libpmix_cb, msg_buf); + msg_buf = NULL; + break; + case CMD_DMDX_REQUEST: { + int *sender_id; + pmix_proc_t proc; + if (NULL == msg_buf) { + abort(); + } + sender_id = (int*)malloc(sizeof(int)); + server_unpack_dmdx(msg_buf, sender_id, &proc); + TEST_VERBOSE(("%d: CMD_DMDX_REQUEST from %d: %s:%d", my_server_id, + *sender_id, proc.nspace, proc.rank)); + rc = PMIx_server_dmodex_request(&proc, _dmdx_cb, (void*)sender_id); + break; + } + case CMD_DMDX_RESPONSE: + TEST_VERBOSE(("%d: CMD_DMDX_RESPONSE", my_server_id)); + server->modex_cbfunc(PMIX_SUCCESS, msg_buf, msg_hdr.size, + server->cbdata, _libpmix_cb, msg_buf); + msg_buf = NULL; + break; + } + if (NULL != msg_buf) { + free(msg_buf); + } +} + +int server_fence_contrib(char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + server_info_t *server; + msg_hdr_t msg_hdr; + int rc = PMIX_SUCCESS; + + if (0 == my_server_id) { + server = my_server_info; + } else { + server = (server_info_t *)pmix_list_get_first(server_list); + } + msg_hdr.cmd = CMD_FENCE_CONTRIB; + msg_hdr.dst_id = 0; + msg_hdr.src_id = my_server_id; + msg_hdr.size = ndata; + server->modex_cbfunc = cbfunc; + server->cbdata = cbdata; + + if (PMIX_SUCCESS != (rc = server_send_msg(&msg_hdr, data, ndata))) { + return PMIX_ERROR; + } + return rc; +} + +static int server_find_id(const char *nspace, int rank) +{ + server_nspace_t *tmp; + + PMIX_LIST_FOREACH(tmp, server_nspace, server_nspace_t) { + if (0 == strcmp(tmp->name, nspace)) { + return tmp->task_map[rank]; + } + } + return -1; +} + +static int server_pack_dmdx(int sender_id, const char *nspace, int rank, + char **buf) +{ + size_t buf_size = sizeof(int) + PMIX_MAX_NSLEN +1 + sizeof(int); + char *ptr; + + *buf = (char*)malloc(buf_size); + ptr = *buf; + + memcpy(ptr, &sender_id, sizeof(int)); + ptr += sizeof(int); + + memcpy(ptr, nspace, PMIX_MAX_NSLEN+1); + ptr += PMIX_MAX_NSLEN +1; + + memcpy(ptr, &rank, sizeof(int)); + ptr += sizeof(int); + + return buf_size; +} + +static void server_unpack_dmdx(char *buf, int *sender, pmix_proc_t *proc) +{ + char *ptr = buf; + + *sender = (int)*ptr; + ptr += sizeof(int); + + memcpy(proc->nspace, ptr, PMIX_MAX_NSLEN +1); + ptr += PMIX_MAX_NSLEN +1; + + proc->rank = (int)*ptr; + ptr += sizeof(int); +} + + +static void _dmdx_cb(int status, char *data, size_t sz, void *cbdata) +{ + msg_hdr_t msg_hdr; + int *sender_id = (int*)cbdata; + + msg_hdr.cmd = CMD_DMDX_RESPONSE; + msg_hdr.src_id = my_server_id; + msg_hdr.size = sz; + msg_hdr.dst_id = *sender_id; + free(sender_id); + + server_send_msg(&msg_hdr, data, sz); +} + +int server_dmdx_get(const char *nspace, int rank, + pmix_modex_cbfunc_t cbfunc, void *cbdata) +{ + server_info_t *server = NULL, *tmp; + msg_hdr_t msg_hdr; + pmix_status_t rc = PMIX_SUCCESS; + char *buf = NULL; + + + if (0 > (msg_hdr.dst_id = server_find_id(nspace, rank))) { + TEST_ERROR(("%d: server cannot found for %s:%d", my_server_id, nspace, rank)); + goto error; + } + + if (0 == my_server_id) { + PMIX_LIST_FOREACH(tmp, server_list, server_info_t) { + if (tmp->idx == msg_hdr.dst_id) { + server = tmp; + break; + } + } + } else { + server = (server_info_t *)pmix_list_get_first(server_list); + } + + if (server == NULL) { + goto error; + } + + msg_hdr.cmd = CMD_DMDX_REQUEST; + msg_hdr.src_id = my_server_id; + msg_hdr.size = server_pack_dmdx(my_server_id, nspace, rank, &buf); + server->modex_cbfunc = cbfunc; + server->cbdata = cbdata; + + if (PMIX_SUCCESS != (rc = server_send_msg(&msg_hdr, buf, msg_hdr.size))) { + rc = PMIX_ERROR; + } + free(buf); + return rc; + +error: + cbfunc(PMIX_ERROR, NULL, 0, cbdata, NULL, 0); + return PMIX_ERROR; +} + +int server_init(test_params *params) +{ + pmix_info_t info[1]; + int rc = PMIX_SUCCESS; + + /* fork/init servers procs */ + if (params->nservers >= 1) { + int i; + server_info_t *server_info = NULL; + server_list = PMIX_NEW(pmix_list_t); + + TEST_VERBOSE(("pmix server %d started PID:%d", my_server_id, getpid())); + for (i = params->nservers - 1; i >= 0; i--) { + pid_t pid; + server_info = PMIX_NEW(server_info_t); + + int fd1[2]; + int fd2[2]; + + pipe(fd1); + pipe(fd2); + + if (0 != i) { + pid = fork(); + if (pid < 0) { + TEST_ERROR(("Fork failed")); + return pid; + } + if (pid == 0) { + server_list = PMIX_NEW(pmix_list_t); + my_server_id = i; + server_info->idx = 0; + server_info->pid = getppid(); + server_info->rd_fd = fd1[0]; + server_info->wr_fd = fd2[1]; + close(fd1[1]); + close(fd2[0]); + PMIX_CONSTRUCT_LOCK(&server_info->lock); + pmix_list_append(server_list, &server_info->super); + break; + } + server_info->idx = i; + server_info->pid = pid; + server_info->wr_fd = fd1[1]; + server_info->rd_fd = fd2[0]; + PMIX_CONSTRUCT_LOCK(&server_info->lock); + close(fd1[0]); + close(fd2[1]); + } else { + my_server_info = server_info; + server_info->pid = getpid(); + server_info->idx = 0; + server_info->rd_fd = fd1[0]; + server_info->wr_fd = fd1[1]; + PMIX_CONSTRUCT_LOCK(&server_info->lock); + close(fd2[0]); + close(fd2[1]); + } + TEST_VERBOSE(("%d: add server %d", my_server_id, server_info->idx)); + pmix_list_append(server_list, &server_info->super); + } + } + /* compute local proc size */ + params->lsize = (params->nprocs % params->nservers) > (uint32_t)my_server_id ? + params->nprocs / params->nservers + 1 : + params->nprocs / params->nservers; + /* setup the server library */ + (void)strncpy(info[0].key, PMIX_SOCKET_MODE, PMIX_MAX_KEYLEN); + info[0].value.type = PMIX_UINT32; + info[0].value.data.uint32 = 0666; + + server_nspace = PMIX_NEW(pmix_list_t); + + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, 1))) { + TEST_ERROR(("Init failed with error %d", rc)); + goto error; + } + + /* register test server read thread */ + if (params->nservers && pmix_list_get_size(server_list)) { + server_info_t *server; + PMIX_LIST_FOREACH(server, server_list, server_info_t) { + server->evread = event_new(pmix_globals.evbase, server->rd_fd, + EV_READ|EV_PERSIST, server_read_cb, server); + event_add(server->evread, NULL); + } + } + + /* register the errhandler */ + PMIx_Register_event_handler(NULL, 0, NULL, 0, + errhandler, errhandler_reg_callbk, NULL); + + if (0 != (rc = server_barrier())) { + goto error; + } + + return PMIX_SUCCESS; + +error: + PMIX_DESTRUCT(server_nspace); + return rc; +} + +int server_finalize(test_params *params) +{ + int rc = PMIX_SUCCESS; + int total_ret = 0; + + if (0 != (rc = server_barrier())) { + total_ret++; + goto exit; + } + + if (0 != my_server_id) { + server_info_t *server = (server_info_t*)pmix_list_get_first(server_list); + remove_server_item(server); + } + + if (params->nservers && 0 == my_server_id) { + int ret; + /* wait for all servers are finished */ + ret = srv_wait_all(10.0); + if (!pmix_list_is_empty(server_list)) { + total_ret += ret; + } + PMIX_LIST_RELEASE(server_list); + TEST_VERBOSE(("SERVER %d FINALIZE PID:%d with status %d", + my_server_id, getpid(), ret)); + if (0 == total_ret) { + TEST_OUTPUT(("Test finished OK!")); + } else { + rc = PMIX_ERROR; + } + } + PMIX_LIST_RELEASE(server_nspace); + + /* finalize the server library */ + if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) { + TEST_ERROR(("Finalize failed with error %d", rc)); + total_ret += rc; + goto exit; + } + +exit: + return total_ret; +} + +int server_launch_clients(int local_size, int univ_size, int base_rank, + test_params *params, char *** client_env, char ***base_argv) +{ + int n; + uid_t myuid; + gid_t mygid; + char *ranks = NULL; + char digit[MAX_DIGIT_LEN]; + int rc; + static int cli_counter = 0; + static int num_ns = 0; + pmix_proc_t proc; + int rank_counter = 0; + server_nspace_t *nspace_item = PMIX_NEW(server_nspace_t); + + TEST_VERBOSE(("%d: lsize: %d, base rank %d, local_size %d, univ_size %d", + my_server_id, + params->lsize, + base_rank, + local_size, + univ_size)); + + TEST_VERBOSE(("Setting job info")); + fill_seq_ranks_array(local_size, base_rank, &ranks); + if (NULL == ranks) { + PMIx_server_finalize(); + TEST_ERROR(("fill_seq_ranks_array failed")); + return PMIX_ERROR; + } + (void)snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, num_ns); + set_namespace(local_size, univ_size, ranks, proc.nspace); + if (NULL != ranks) { + free(ranks); + } + /* add namespace entry */ + nspace_item->ntasks = univ_size; + nspace_item->ltasks = local_size; + nspace_item->task_map = (int*)malloc(sizeof(int) * univ_size); + memset(nspace_item->task_map, -1, sizeof(int)*univ_size); + strcpy(nspace_item->name, proc.nspace); + pmix_list_append(server_nspace, &nspace_item->super); + for (n = 0; n < local_size; n++) { + proc.rank = base_rank + n; + nspace_item->task_map[proc.rank] = my_server_id; + } + + server_send_procs(); + + myuid = getuid(); + mygid = getgid(); + + /* fork/exec the test */ + for (n = 0; n < local_size; n++) { + proc.rank = base_rank + rank_counter; + if (PMIX_SUCCESS != (rc = PMIx_server_setup_fork(&proc, client_env))) {//n + TEST_ERROR(("Server fork setup failed with error %d", rc)); + PMIx_server_finalize(); + cli_kill_all(); + return rc; + } + if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, NULL, NULL))) {//n + TEST_ERROR(("Server fork setup failed with error %d", rc)); + PMIx_server_finalize(); + cli_kill_all(); + return 0; + } + + cli_info[cli_counter].pid = fork(); + if (cli_info[cli_counter].pid < 0) { + TEST_ERROR(("Fork failed")); + PMIx_server_finalize(); + cli_kill_all(); + return 0; + } + cli_info[cli_counter].rank = proc.rank;//n + cli_info[cli_counter].ns = strdup(proc.nspace); + + char **client_argv = pmix_argv_copy(*base_argv); + + /* add two last arguments: -r */ + sprintf(digit, "%d", proc.rank); + pmix_argv_append_nosize(&client_argv, "-r"); + pmix_argv_append_nosize(&client_argv, digit); + + pmix_argv_append_nosize(&client_argv, "-s"); + pmix_argv_append_nosize(&client_argv, proc.nspace); + + sprintf(digit, "%d", univ_size); + pmix_argv_append_nosize(&client_argv, "--ns-size"); + pmix_argv_append_nosize(&client_argv, digit); + + sprintf(digit, "%d", num_ns); + pmix_argv_append_nosize(&client_argv, "--ns-id"); + pmix_argv_append_nosize(&client_argv, digit); + + sprintf(digit, "%d", 0); + pmix_argv_append_nosize(&client_argv, "--base-rank"); + pmix_argv_append_nosize(&client_argv, digit); + + if (cli_info[cli_counter].pid == 0) { + if( !TEST_VERBOSE_GET() ){ + // Hide clients stdout + if (NULL == freopen("/dev/null","w", stdout)) { + return 0; + } + } + execve(params->binary, client_argv, *client_env); + /* Does not return */ + TEST_ERROR(("execve() failed")); + return 0; + } + cli_info[cli_counter].state = CLI_FORKED; + + pmix_argv_free(client_argv); + + cli_counter++; + rank_counter++; + } + num_ns++; + return rank_counter; +} diff --git a/opal/mca/pmix/pmix3x/pmix/test/test_server.h b/opal/mca/pmix/pmix3x/pmix/test/test_server.h new file mode 100644 index 0000000000..09767ea56f --- /dev/null +++ b/opal/mca/pmix/pmix3x/pmix/test/test_server.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2018 Mellanox Technologies, Inc. + * All rights reserved. + * + * Copyright (c) 2018 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#ifndef TEST_SERVER_C +#define TEST_SERVER_C + +#include "pmix_server.h" +#include "test_common.h" + + +typedef enum { + CMD_BARRIER_REQUEST, + CMD_BARRIER_RESPONSE, + CMD_FENCE_CONTRIB, + CMD_FENCE_COMPLETE, + CMD_DMDX_REQUEST, + CMD_DMDX_RESPONSE +} server_cmd_t; + +typedef struct { + int dst_id; + int src_id; + int cmd; + size_t size; +} msg_hdr_t; + +struct server_info_t +{ + pmix_list_item_t super; + pid_t pid; + int idx; + int rd_fd; + int wr_fd; + pmix_event_t *evread; + pmix_lock_t lock; + pmix_modex_cbfunc_t modex_cbfunc; + void *cbdata; +}; +typedef struct server_info_t server_info_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(server_info_t); + +struct server_nspace_t +{ + pmix_list_item_t super; + char name[PMIX_MAX_NSLEN+1]; + size_t ntasks; /* total number of tasks in this namespace */ + size_t ltasks; /* local */ + int *task_map; +}; +typedef struct server_nspace_t server_nspace_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(server_nspace_t); + +extern int my_server_id; +extern pmix_list_t *server_list; +extern server_info_t *my_server_info; +extern pmix_list_t *server_nspace; + +int server_init(test_params *params); +int server_finalize(test_params *params); +int server_barrier(void); +int server_fence_contrib(char *data, size_t ndata, + pmix_modex_cbfunc_t cbfunc, void *cbdata); +int server_dmdx_get(const char *nspace, int rank, + pmix_modex_cbfunc_t cbfunc, void *cbdata); +int server_launch_clients(int local_size, int univ_size, int base_rank, + test_params *params, char *** client_env, char ***base_argv); + + +#endif // TEST_SERVER_C + diff --git a/opal/mca/pmix/pmix3x/pmix/test/utils.c b/opal/mca/pmix/pmix3x/pmix/test/utils.c index 5fb1a0e78b..d6cd31b541 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/utils.c +++ b/opal/mca/pmix/pmix3x/pmix/test/utils.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015-2017 Mellanox Technologies, Inc. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -16,83 +16,7 @@ #include "test_common.h" #include "pmix_server.h" #include "cli_stages.h" - -static void release_cb(pmix_status_t status, void *cbdata) -{ - int *ptr = (int*)cbdata; - *ptr = 0; -} - -static void fill_seq_ranks_array(size_t nprocs, int base_rank, char **ranks) -{ - uint32_t i; - int len = 0, max_ranks_len; - if (0 >= nprocs) { - return; - } - max_ranks_len = nprocs * (MAX_DIGIT_LEN+1); - *ranks = (char*) malloc(max_ranks_len); - for (i = 0; i < nprocs; i++) { - len += snprintf(*ranks + len, max_ranks_len-len-1, "%d", i+base_rank); - if (i != nprocs-1) { - len += snprintf(*ranks + len, max_ranks_len-len-1, "%c", ','); - } - } - if (len >= max_ranks_len-1) { - free(*ranks); - *ranks = NULL; - TEST_ERROR(("Not enough allocated space for global ranks array.")); - } -} - -static void set_namespace(int nprocs, char *ranks, char *name) -{ - size_t ninfo; - pmix_info_t *info; - ninfo = 8; - char *regex, *ppn; - - PMIX_INFO_CREATE(info, ninfo); - (void)strncpy(info[0].key, PMIX_UNIV_SIZE, PMIX_MAX_KEYLEN); - info[0].value.type = PMIX_UINT32; - info[0].value.data.uint32 = nprocs; - - (void)strncpy(info[1].key, PMIX_SPAWNED, PMIX_MAX_KEYLEN); - info[1].value.type = PMIX_UINT32; - info[1].value.data.uint32 = 0; - - (void)strncpy(info[2].key, PMIX_LOCAL_SIZE, PMIX_MAX_KEYLEN); - info[2].value.type = PMIX_UINT32; - info[2].value.data.uint32 = nprocs; - - (void)strncpy(info[3].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN); - info[3].value.type = PMIX_STRING; - info[3].value.data.string = strdup(ranks); - - PMIx_generate_regex(NODE_NAME, ®ex); - (void)strncpy(info[4].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN); - info[4].value.type = PMIX_STRING; - info[4].value.data.string = regex; - - PMIx_generate_ppn(ranks, &ppn); - (void)strncpy(info[5].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN); - info[5].value.type = PMIX_STRING; - info[5].value.data.string = ppn; - - (void)strncpy(info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN); - info[6].value.type = PMIX_UINT32; - info[6].value.data.uint32 = nprocs; - - (void)strncpy(info[7].key, PMIX_APPNUM, PMIX_MAX_KEYLEN); - info[7].value.type = PMIX_UINT32; - info[7].value.data.uint32 = getpid (); - - int in_progress = 1, rc; - if (PMIX_SUCCESS == (rc = PMIx_server_register_nspace(name, nprocs, info, ninfo, release_cb, &in_progress))) { - PMIX_WAIT_FOR_COMPLETION(in_progress); - } - PMIX_INFO_FREE(info, ninfo); -} +#include "test_server.h" void set_client_argv(test_params *params, char ***argv) { @@ -170,100 +94,3 @@ void set_client_argv(test_params *params, char ***argv) pmix_argv_append_nosize(argv, params->gds_mode); } } - -int launch_clients(int num_procs, char *binary, char *** client_env, char ***base_argv) -{ - int n; - uid_t myuid; - gid_t mygid; - char *ranks = NULL; - char digit[MAX_DIGIT_LEN]; - int rc; - static int counter = 0; - static int num_ns = 0; - pmix_proc_t proc; - - TEST_VERBOSE(("Setting job info")); - fill_seq_ranks_array(num_procs, counter, &ranks); - if (NULL == ranks) { - PMIx_server_finalize(); - TEST_ERROR(("fill_seq_ranks_array failed")); - return PMIX_ERROR; - } - (void)snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s-%d", TEST_NAMESPACE, num_ns); - set_namespace(num_procs, ranks, proc.nspace); - if (NULL != ranks) { - free(ranks); - } - - myuid = getuid(); - mygid = getgid(); - - /* fork/exec the test */ - for (n = 0; n < num_procs; n++) { - proc.rank = counter; - if (PMIX_SUCCESS != (rc = PMIx_server_setup_fork(&proc, client_env))) {//n - TEST_ERROR(("Server fork setup failed with error %d", rc)); - PMIx_server_finalize(); - cli_kill_all(); - return rc; - } - if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, NULL, NULL))) {//n - TEST_ERROR(("Server fork setup failed with error %d", rc)); - PMIx_server_finalize(); - cli_kill_all(); - return rc; - } - - cli_info[counter].pid = fork(); - if (cli_info[counter].pid < 0) { - TEST_ERROR(("Fork failed")); - PMIx_server_finalize(); - cli_kill_all(); - return -1; - } - cli_info[counter].rank = counter;//n - cli_info[counter].ns = strdup(proc.nspace); - - char **client_argv = pmix_argv_copy(*base_argv); - - /* add two last arguments: -r */ - sprintf(digit, "%d", counter);//n - pmix_argv_append_nosize(&client_argv, "-r"); - pmix_argv_append_nosize(&client_argv, digit); - - pmix_argv_append_nosize(&client_argv, "-s"); - pmix_argv_append_nosize(&client_argv, proc.nspace); - - sprintf(digit, "%d", num_procs); - pmix_argv_append_nosize(&client_argv, "--ns-size"); - pmix_argv_append_nosize(&client_argv, digit); - - sprintf(digit, "%d", num_ns); - pmix_argv_append_nosize(&client_argv, "--ns-id"); - pmix_argv_append_nosize(&client_argv, digit); - - sprintf(digit, "%d", (counter-n)); - pmix_argv_append_nosize(&client_argv, "--base-rank"); - pmix_argv_append_nosize(&client_argv, digit); - - if (cli_info[counter].pid == 0) { - if( !TEST_VERBOSE_GET() ){ - // Hide clients stdout - if (NULL == freopen("/dev/null","w", stdout)) { - exit(1); - } - } - execve(binary, client_argv, *client_env); - /* Does not return */ - exit(0); - } - cli_info[counter].state = CLI_FORKED; - - pmix_argv_free(client_argv); - - counter++; - } - num_ns++; - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix3x/pmix/test/utils.h b/opal/mca/pmix/pmix3x/pmix/test/utils.h index bbeebaa2f7..d6856dd7a1 100644 --- a/opal/mca/pmix/pmix3x/pmix/test/utils.h +++ b/opal/mca/pmix/pmix3x/pmix/test/utils.h @@ -1,6 +1,6 @@ /* - * Copyright (c) 2015 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2018 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -19,4 +19,3 @@ #include "test_common.h" void set_client_argv(test_params *params, char ***argv); -int launch_clients(int num_procs, char *binary, char *** client_env, char ***client_argv);