From fbed2d794a2d48fa9979b7b65199367914b009d2 Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Tue, 6 Dec 2016 15:05:34 -0800 Subject: [PATCH] Update to latest PMIx master + PTL branch Update the usock component to disable it Signed-off-by: Ralph Castain --- opal/mca/pmix/pmix2x/pmix/VERSION | 4 +- .../pmix/pmix2x/pmix/include/pmix_common.h | 24 +- opal/mca/pmix/pmix2x/pmix/src/Makefile.am | 1 - .../pmix/pmix2x/pmix/src/client/pmix_client.c | 431 +----- .../pmix/src/client/pmix_client_connect.c | 22 +- .../pmix/src/client/pmix_client_fence.c | 14 +- .../pmix2x/pmix/src/client/pmix_client_get.c | 23 +- .../pmix2x/pmix/src/client/pmix_client_ops.h | 1 - .../pmix2x/pmix/src/client/pmix_client_pub.c | 41 +- .../pmix/src/client/pmix_client_spawn.c | 15 +- .../pmix/pmix2x/pmix/src/common/pmix_log.c | 11 +- .../pmix/pmix2x/pmix/src/common/pmix_query.c | 11 +- .../pmix/src/event/pmix_event_notification.c | 11 +- .../pmix/src/event/pmix_event_registration.c | 17 +- .../pmix2x/pmix/src/include/pmix_globals.c | 79 +- .../pmix2x/pmix/src/include/pmix_globals.h | 91 +- .../pmix/src/mca/base/pmix_mca_base_var.c | 4 +- opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h | 2 +- .../pmix/pmix2x/pmix/src/mca/psec/base/base.h | 12 + .../pmix/src/mca/psec/base/psec_base_fns.c | 84 +- .../pmix/src/mca/psec/base/psec_base_frame.c | 7 + .../pmix/src/mca/psec/munge/psec_munge.c | 25 +- .../pmix/src/mca/psec/native/psec_native.c | 174 ++- .../pmix2x/pmix/src/mca/psec/none/Makefile.am | 50 + .../pmix2x/pmix/src/mca/psec/none/psec_none.c | 63 + .../pmix2x/pmix/src/mca/psec/none/psec_none.h | 27 + .../src/mca/psec/none/psec_none_component.c | 91 ++ opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h | 70 +- .../pmix/pmix2x/pmix/src/mca/ptl/Makefile.am | 44 + .../pmix/src/mca/ptl/base/Makefile.include | 35 + .../pmix/pmix2x/pmix/src/mca/ptl/base/base.h | 123 ++ .../pmix/src/mca/ptl/base/ptl_base_connect.c | 249 ++++ .../pmix/src/mca/ptl/base/ptl_base_frame.c | 220 +++ .../pmix/src/mca/ptl/base/ptl_base_listener.c | 281 ++++ .../pmix/src/mca/ptl/base/ptl_base_select.c | 94 ++ .../ptl/base/ptl_base_sendrecv.c} | 142 +- .../pmix/src/mca/ptl/base/ptl_base_stubs.c | 107 ++ opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h | 199 +++ .../pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h | 257 ++++ .../pmix2x/pmix/src/mca/ptl/tcp/Makefile.am | 50 + .../pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c | 732 ++++++++++ .../pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h | 56 + .../pmix/src/mca/ptl/tcp/ptl_tcp_component.c | 1192 +++++++++++++++++ .../pmix2x/pmix/src/mca/ptl/usock/Makefile.am | 50 + .../pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c | 358 +++++ .../pmix2x/pmix/src/mca/ptl/usock/ptl_usock.h | 47 + .../src/mca/ptl/usock/ptl_usock_component.c | 621 +++++++++ .../pmix2x/pmix/src/runtime/pmix_finalize.c | 14 +- .../pmix/pmix2x/pmix/src/runtime/pmix_init.c | 68 +- .../pmix/pmix2x/pmix/src/runtime/pmix_rte.h | 3 +- .../pmix2x/pmix/src/server/Makefile.include | 3 +- .../pmix/pmix2x/pmix/src/server/pmix_server.c | 328 +---- .../pmix2x/pmix/src/server/pmix_server_get.c | 1 - .../pmix/src/server/pmix_server_listener.c | 972 -------------- .../pmix2x/pmix/src/server/pmix_server_ops.c | 64 - .../pmix2x/pmix/src/server/pmix_server_ops.h | 92 -- .../mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c | 457 +------ .../pmix/pmix2x/pmix/src/usock/Makefile.am | 17 - opal/mca/pmix/pmix2x/pmix/src/usock/usock.c | 348 ----- opal/mca/pmix/pmix2x/pmix/src/usock/usock.h | 199 --- .../pmix2x/pmix/src/util/Makefile.include | 6 +- opal/mca/pmix/pmix2x/pmix/src/util/error.c | 4 + opal/mca/pmix/pmix2x/pmix/src/util/error.h | 2 + .../pmix/pmix2x/pmix/src/util/parse_options.c | 143 ++ .../pmix/pmix2x/pmix/src/util/parse_options.h | 37 + opal/mca/pmix/pmix2x/pmix/src/util/pif.c | 5 +- opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h | 4 +- opal/mca/pmix/pmix2x/pmix/test/cli_stages.c | 2 +- opal/mca/pmix/pmix2x/pmix/test/cli_stages.h | 2 +- .../pmix2x/pmix/test/pmix_client_otheruser.sh | 0 .../pmix/pmix2x/pmix/test/simple/simpdmodex.c | 8 +- .../pmix/pmix2x/pmix/test/simple/simptest.c | 92 +- opal/mca/pmix/pmix2x/pmix/test/test_cd.c | 7 + opal/mca/pmix/pmix2x/pmix/test/test_common.h | 1 - opal/mca/pmix/pmix2x/pmix/test/test_spawn.c | 7 + opal/mca/pmix/pmix_types.h | 12 + orte/mca/ess/singleton/ess_singleton_module.c | 2 +- orte/orted/orted_main.c | 5 +- orte/orted/pmix/pmix_server.c | 6 + 79 files changed, 6043 insertions(+), 3130 deletions(-) create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/Makefile.am create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/Makefile.include create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_connect.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_listener.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_select.c rename opal/mca/pmix/pmix2x/pmix/src/{usock/usock_sendrecv.c => mca/ptl/base/ptl_base_sendrecv.c} (79%) create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c delete mode 100644 opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_listener.c delete mode 100644 opal/mca/pmix/pmix2x/pmix/src/usock/Makefile.am delete mode 100644 opal/mca/pmix/pmix2x/pmix/src/usock/usock.c delete mode 100644 opal/mca/pmix/pmix2x/pmix/src/usock/usock.h create mode 100644 opal/mca/pmix/pmix2x/pmix/src/util/parse_options.c create mode 100644 opal/mca/pmix/pmix2x/pmix/src/util/parse_options.h mode change 100644 => 100755 opal/mca/pmix/pmix2x/pmix/test/pmix_client_otheruser.sh diff --git a/opal/mca/pmix/pmix2x/pmix/VERSION b/opal/mca/pmix/pmix2x/pmix/VERSION index 9602917ea8..959426eaf5 100644 --- a/opal/mca/pmix/pmix2x/pmix/VERSION +++ b/opal/mca/pmix/pmix2x/pmix/VERSION @@ -30,7 +30,7 @@ greek= # command, or with the date (if "git describe" fails) in the form of # "date". -repo_rev=gitc8f26f9 +repo_rev=git22b754e # If tarball_version is not empty, it is used as the version string in # the tarball filename, regardless of all other versions listed in @@ -44,7 +44,7 @@ tarball_version= # The date when this release was created -date="Dec 02, 2016" +date="Dec 06, 2016" # The shared library version of each of PMIx's public libraries. # These versions are maintained in accordance with the "Library diff --git a/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h b/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h index 67e22d1aa8..17869a412f 100644 --- a/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h @@ -109,13 +109,15 @@ typedef uint32_t pmix_rank_t; // of the internal progress thread #define PMIX_SERVER_TOOL_SUPPORT "pmix.srvr.tool" // (bool) The host RM wants to declare itself as willing to // accept tool connection requests +#define PMIX_SERVER_REMOTE_CONNECTIONS "pmix.srvr.remote" // (bool) Allow connections from remote tools (do not use loopback device) #define PMIX_SERVER_SYSTEM_SUPPORT "pmix.srvr.sys" // (bool) The host RM wants to declare itself as being the local // system server for PMIx connection requests #define PMIX_SERVER_PIDINFO "pmix.srvr.pidinfo" // (pid_t) pid of the target server +#define PMIX_SERVER_HOSTNAME "pmix.srvr.host" // (char*) node where target server is located #define PMIX_SERVER_TMPDIR "pmix.srvr.tmpdir" // (char*) temp directory where PMIx server will place - // client rendezvous points + // client rendezvous points and contact info #define PMIX_SYSTEM_TMPDIR "pmix.sys.tmpdir" // (char*) temp directory for this system, where PMIx - // server will place tool rendezvous points + // server will place tool rendezvous points and contact info #define PMIX_CONNECT_TO_SYSTEM "pmix.cnct.sys" // (bool) The requestor requires that a connection be made only to // a local system-level PMIx server #define PMIX_CONNECT_SYSTEM_FIRST "pmix.cnct.sys.first" // (bool) Preferentially look for a system-level PMIx server first @@ -124,10 +126,22 @@ typedef uint32_t pmix_rank_t; #define PMIX_USERID "pmix.euid" // (uint32_t) effective user id #define PMIX_GRPID "pmix.egid" // (uint32_t) effective group id #define PMIX_DSTPATH "pmix.dstpath" // (char*) path to dstore files +#define PMIX_VERSION_INFO "pmix.version" // (char*) PMIx version of contactor -/* attributes for the rendezvous socket */ + +/* attributes for the USOCK rendezvous socket */ +#define PMIX_USOCK_DISABLE "pmix.usock.disable" // (bool) disable legacy usock support #define PMIX_SOCKET_MODE "pmix.sockmode" // (uint32_t) POSIX mode_t (9 bits valid) +/* attributes for TCP connections */ +#define PMIX_TCP_URI "pmix.tcp.uri" // (char*) URI of server to connect to +#define PMIX_TCP_IF_INCLUDE "pmix.tcp.ifinclude" // (char*) comma-delimited list of devices and/or CIDR notation +#define PMIX_TCP_IF_EXCLUDE "pmix.tcp.ifexclude" // (char*) comma-delimited list of devices and/or CIDR notation +#define PMIX_TCP_IPV4_PORT "pmix.tcp.ipv4" // (int) IPv4 port to be used +#define PMIX_TCP_IPV6_PORT "pmix.tcp.ipv6" // (int) IPv6 port to be used +#define PMIX_TCP_DISABLE_IPV4 "pmix.tcp.disipv4" // (bool) true to disable IPv4 family +#define PMIX_TCP_DISABLE_IPV6 "pmix.tcp.disipv6" // (bool) true to disable IPv6 family + /* general proc-level attributes */ #define PMIX_CPUSET "pmix.cpuset" // (char*) hwloc bitmap applied to proc upon launch #define PMIX_CREDENTIAL "pmix.cred" // (char*) security credential assigned to proc @@ -267,6 +281,10 @@ typedef uint32_t pmix_rank_t; #define PMIX_LOG_STDOUT "pmix.log.stdout" // (bool) log data to stdout #define PMIX_LOG_SYSLOG "pmix.log.syslog" // (bool) log data to syslog - defaults to ERROR priority unless // modified by directive +/* debugger attributes */ +#define PMIX_SPAWN_UNDER_DEBUGGER "pmix.dbg.pause" // (bool) job is being spawned under debugger - instruct it to pause on start +#define PMIX_JOB_BEING_DEBUGGED "pmix.dbg.job" // (char*) nspace of the job to be debugged - the RM/PMIx server are + // to provide the job-level info of that job to each debugger daemon /**** PROCESS STATE DEFINITIONS ****/ typedef uint8_t pmix_proc_state_t; diff --git a/opal/mca/pmix/pmix2x/pmix/src/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/Makefile.am index c5d4828316..97ea7b3de9 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/src/Makefile.am @@ -72,7 +72,6 @@ include runtime/Makefile.include include tool/Makefile.include include common/Makefile.include include buffer_ops/Makefile.am -include usock/Makefile.am if WANT_DSTORE include sm/Makefile.include include dstore/Makefile.include diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c index 9b58c2672b..44cafb5a0d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c @@ -65,7 +65,7 @@ #include "src/util/output.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #include "src/include/pmix_globals.h" #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) #include "src/dstore/pmix_dstore.h" @@ -76,15 +76,14 @@ #define PMIX_MAX_RETRIES 10 -static pmix_status_t usock_connect(struct sockaddr *address, int *fd); - static void _notify_complete(pmix_status_t status, void *cbdata) { pmix_event_chain_t *chain = (pmix_event_chain_t*)cbdata; PMIX_RELEASE(chain); } -static void pmix_client_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, +static void pmix_client_notify_recv(struct pmix_peer_t *peer, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_status_t rc; @@ -161,20 +160,21 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t * pmix_client_globals_t pmix_client_globals = {{{0}}}; /* callback for wait completion */ -static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { - pmix_cb_t *cb = (pmix_cb_t*)cbdata; + volatile bool *active = (volatile bool*)cbdata; pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:client recv callback activated with %d bytes", - (NULL == buf) ? -1 : (int)buf->bytes_used); + "pmix:client wait_cbfunc received"); - cb->active = false; + *active = false; } /* callback to receive job info */ -static void job_data(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void job_data(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_status_t rc; @@ -201,49 +201,6 @@ static void job_data(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, cb->active = false; } -static pmix_status_t connect_to_server(struct sockaddr_un *address, void *cbdata) -{ - int sd; - pmix_status_t ret; - pmix_cmd_t cmd = PMIX_REQ_CMD; - pmix_buffer_t *req; - - if (PMIX_SUCCESS != (ret=usock_connect((struct sockaddr *)address, &sd))) { - PMIX_ERROR_LOG(ret); - return ret; - } - pmix_client_globals.myserver.sd = sd; - /* setup recv event */ - event_assign(&pmix_client_globals.myserver.recv_event, - pmix_globals.evbase, - pmix_client_globals.myserver.sd, - EV_READ | EV_PERSIST, - pmix_usock_recv_handler, &pmix_client_globals.myserver); - event_add(&pmix_client_globals.myserver.recv_event, 0); - pmix_client_globals.myserver.recv_ev_active = true; - - /* setup send event */ - event_assign(&pmix_client_globals.myserver.send_event, - pmix_globals.evbase, - pmix_client_globals.myserver.sd, - EV_WRITE|EV_PERSIST, - pmix_usock_send_handler, &pmix_client_globals.myserver); - pmix_client_globals.myserver.send_ev_active = false; - - /* send a request for our job info - we do this as a non-blocking - * transaction because some systems cannot handle very large - * blocking operations and error out if we try them. */ - req = PMIX_NEW(pmix_buffer_t); - if (PMIX_SUCCESS != (ret = pmix_bfrop.pack(req, &cmd, 1, PMIX_CMD))) { - PMIX_ERROR_LOG(ret); - PMIX_RELEASE(req); - return ret; - } - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, req, job_data, cbdata); - - return PMIX_SUCCESS; -} - PMIX_EXPORT const char* PMIx_Get_version(void) { return pmix_version_string; @@ -252,11 +209,12 @@ PMIX_EXPORT const char* PMIx_Get_version(void) PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_info_t info[], size_t ninfo) { - char **uri, *evar; - int rc; - struct sockaddr_un address; + char *evar; + pmix_status_t rc; pmix_nspace_t *nsptr; pmix_cb_t cb; + pmix_buffer_t *req; + pmix_cmd_t cmd = PMIX_REQ_CMD; if (NULL == proc) { return PMIX_ERR_BAD_PARAM; @@ -293,7 +251,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, pmix_output_verbose(2, pmix_globals.debug_output, "pmix: init called"); - /* we require the nspace */ + /* we require our nspace */ if (NULL == (evar = getenv("PMIX_NAMESPACE"))) { /* let the caller know that the server isn't available yet */ return PMIX_ERR_INVALID_NAMESPACE; @@ -306,37 +264,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, (void)strncpy(nsptr->nspace, evar, PMIX_MAX_NSLEN); pmix_list_append(&pmix_globals.nspaces, &nsptr->super); - /* if we don't have a path to the daemon rendezvous point, - * then we need to return an error */ - if (NULL == (evar = getenv("PMIX_SERVER_URI"))) { - /* let the caller know that the server isn't available */ - return PMIX_ERR_SERVER_NOT_AVAIL; - } - uri = pmix_argv_split(evar, ':'); - if (3 != pmix_argv_count(uri)) { - pmix_argv_free(uri); - return PMIX_ERROR; - } - - /* set the server nspace */ - pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t); - pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t); - (void)strncpy(pmix_client_globals.myserver.info->nptr->nspace, uri[0], PMIX_MAX_NSLEN); - - /* set the server rank */ - pmix_client_globals.myserver.info->rank = strtoull(uri[1], NULL, 10); - - /* setup the path to the daemon rendezvous point */ - memset(&address, 0, sizeof(struct sockaddr_un)); - address.sun_family = AF_UNIX; - snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s", uri[2]); - /* if the rendezvous file doesn't exist, that's an error */ - if (0 != access(uri[2], R_OK)) { - pmix_argv_free(uri); - return PMIX_ERR_NOT_FOUND; - } - pmix_argv_free(uri); - /* we also require our rank */ if (NULL == (evar = getenv("PMIX_RANK"))) { /* let the caller know that the server isn't available yet */ @@ -348,21 +275,45 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc, } pmix_globals.pindex = -1; - /* setup the support */ + /* select our psec compat module - the selection will be based + * on the corresponding envars that should have been passed + * to us at launch */ + evar = getenv("PMIX_SECURITY_MODE"); + if (PMIX_SUCCESS != (rc = pmix_psec.assign_module(pmix_globals.mypeer, evar))) { + return PMIX_ERR_INIT; + } + /* the server will be using the same */ + pmix_client_globals.myserver.compat.psec = pmix_globals.mypeer->compat.psec; + + /* setup the shared memory support */ #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) if (PMIX_SUCCESS != (rc = pmix_dstore_init(NULL, 0))) { return PMIX_ERR_DATA_VALUE_NOT_FOUND; } #endif /* PMIX_ENABLE_DSTORE */ - /* setup an object to track server connection */ + /* connect to the server */ + if (PMIX_SUCCESS != (rc = pmix_ptl.connect_to_peer(&pmix_client_globals.myserver, info, ninfo))){ + return rc; + } + + /* send a request for our job info - we do this as a non-blocking + * transaction because some systems cannot handle very large + * blocking operations and error out if we try them. */ + req = PMIX_NEW(pmix_buffer_t); + if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(req, &cmd, 1, PMIX_CMD))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(req); + return rc; + } + /* send to the server */ PMIX_CONSTRUCT(&cb, pmix_cb_t); cb.active = true; - /* connect to the server - returns job info if successful */ - if (PMIX_SUCCESS != (rc = connect_to_server(&address, &cb))){ + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, req, job_data, (void*)&cb))){ PMIX_DESTRUCT(&cb); return rc; } + /* wait for the data to return */ PMIX_WAIT_FOR_COMPLETION(cb.active); rc = cb.status; PMIX_DESTRUCT(&cb); @@ -384,10 +335,10 @@ PMIX_EXPORT int PMIx_Initialized(void) PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) { pmix_buffer_t *msg; - pmix_cb_t *cb; pmix_cmd_t cmd = PMIX_FINALIZE_CMD; pmix_status_t rc; size_t n; + volatile bool active; if (1 != pmix_globals.init_cntr) { --pmix_globals.init_cntr; @@ -430,21 +381,19 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo) return rc; } - /* create a callback object as we need to pass it to the - * recv routine so we know which callback to use when - * the return message is recvd */ - cb = PMIX_NEW(pmix_cb_t); - cb->active = true; pmix_output_verbose(2, pmix_globals.debug_output, "pmix:client sending finalize sync to server"); - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); + /* send to the server */ + active = true;; + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, + wait_cbfunc, (void*)&active))){ + return rc; + } /* wait for the ack to return */ - PMIX_WAIT_FOR_COMPLETION(cb->active); - PMIX_RELEASE(cb); + PMIX_WAIT_FOR_COMPLETION(active); pmix_output_verbose(2, pmix_globals.debug_output, "pmix:client finalize sync received"); } @@ -471,7 +420,7 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int flag, const char msg[], pmix_buffer_t *bfr; pmix_cmd_t cmd = PMIX_ABORT_CMD; pmix_status_t rc; - pmix_cb_t *cb; + pmix_ptl_sr_t cb; pmix_output_verbose(2, pmix_globals.debug_output, "pmix:client abort called"); @@ -523,15 +472,20 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int flag, const char msg[], /* create a callback object as we need to pass it to the * recv routine so we know which callback to use when * the return message is recvd */ - cb = PMIX_NEW(pmix_cb_t); - cb->active = true; + PMIX_CONSTRUCT(&cb, pmix_ptl_sr_t); + cb.active = true; + cb.cbfunc = wait_cbfunc; - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, bfr, wait_cbfunc, cb); + /* send to the server */ + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, bfr, + wait_cbfunc, &cb))){ + PMIX_DESTRUCT(&cb); + return rc; + } /* wait for the release */ - PMIX_WAIT_FOR_COMPLETION(cb->active); - PMIX_RELEASE(cb); + PMIX_WAIT_FOR_COMPLETION(cb.active); + PMIX_DESTRUCT(&cb); return PMIX_SUCCESS; } @@ -672,14 +626,17 @@ static void _commitfn(int sd, short args, void *cbdata) PMIX_RELEASE(pmix_globals.cache_remote); } - /* push the message into our event base to send to the server - always - * send, even if we have nothing to contribute, so the server knows + /* always send, even if we have nothing to contribute, so the server knows * that we contributed whatever we had */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msgout, NULL, NULL); + if (PMIX_SUCCESS == (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msgout, + wait_cbfunc, (void*)&cb->active))){ + cb->pstatus = PMIX_SUCCESS; + return; + } - done: - cb->pstatus = rc; - cb->active = false; + done: + cb->pstatus = rc; + cb->active = false; } PMIX_EXPORT pmix_status_t PMIx_Commit(void) @@ -874,247 +831,3 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist return rc; } - - -static pmix_status_t send_connect_ack(int sd) -{ - char *msg; - pmix_usock_hdr_t hdr; - size_t sdsize=0, csize=0; - char *cred = NULL; - char *sec; - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: SEND CONNECT ACK"); - - /* setup the header */ - memset(&hdr, 0, sizeof(pmix_usock_hdr_t)); - hdr.pindex = -1; - hdr.tag = UINT32_MAX; - - /* reserve space for the nspace and rank info */ - sdsize = strlen(pmix_globals.myid.nspace) + 1 + sizeof(int); - - /* a security module was assigned to us during rte_init based - * on a list of available security modules provided by our - * local PMIx server. Now use that module to - * get a credential, if the security system provides one. Not - * every psec module will do so, thus we must first check */ - if (NULL != pmix_globals.mypeer->compat.psec->create_cred) { - if (NULL == (cred = pmix_globals.mypeer->compat.psec->create_cred())) { - /* an error occurred - we cannot continue */ - return PMIX_ERR_INVALID_CRED; - } - csize = strlen(cred) + 1; // must NULL terminate the string! - } - - /* add our active bfrops and sec module info, and what type - * of buffers we are using */ - // bfrop = pmix_globals.mypeer->compat.bfrops->name; - sec = pmix_globals.mypeer->compat.psec->name; - - /* set the number of bytes to be read beyond the header */ - hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 + csize; // must NULL terminate the VERSION string! - - /* create a space for our message */ - sdsize = (sizeof(hdr) + hdr.nbytes); - if (NULL == (msg = (char*)malloc(sdsize))) { - if (NULL != cred) { - free(cred); - } - return PMIX_ERR_OUT_OF_RESOURCE; - } - memset(msg, 0, sdsize); - - /* load the message */ - csize=0; - memcpy(msg, &hdr, sizeof(pmix_usock_hdr_t)); - csize += sizeof(pmix_usock_hdr_t); - memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); - csize += strlen(pmix_globals.myid.nspace)+1; - memcpy(msg+csize, &pmix_globals.myid.rank, sizeof(int)); - csize += sizeof(int); - memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION)); - csize += strlen(PMIX_VERSION)+1; - // memcpy(msg+csize, bfrop, strlen(bfrop)); - // csize += strlen(bfrop)+1; - memcpy(msg+csize, sec, strlen(sec)); - csize += strlen(sec)+1; - // memcpy(msg+csize, &pmix_globals.mypeer->comm.type, sizeof(pmix_bfrop_buffer_type_t)); - // csize += sizeof(pmix_bfrop_buffer_type_t); - if (NULL != cred) { - memcpy(msg+csize, cred, strlen(cred)); // leaves last position in msg set to NULL - } - - if (PMIX_SUCCESS != pmix_usock_send_blocking(sd, msg, sdsize)) { - free(msg); - if (NULL != cred) { - free(cred); - } - return PMIX_ERR_UNREACH; - } - free(msg); - if (NULL != cred) { - free(cred); - } - return PMIX_SUCCESS; -} - -/* we receive a connection acknowledgement from the server, - * consisting of nothing more than a status report. If success, - * then we initiate authentication method */ - static pmix_status_t recv_connect_ack(int sd) - { - pmix_status_t reply; - pmix_status_t rc; - struct timeval tv, save; - pmix_socklen_t sz; - bool sockopt = true; - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: RECV CONNECT ACK FROM SERVER"); - - /* get the current timeout value so we can reset to it */ - sz = sizeof(save); - if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) { - if (ENOPROTOOPT == errno) { - sockopt = false; - } else { - return PMIX_ERR_UNREACH; - } - } else { - /* set a timeout on the blocking recv so we don't hang */ - tv.tv_sec = 2; - tv.tv_usec = 0; - if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); - return PMIX_ERR_UNREACH; - } - } - - /* receive the status reply */ - rc = pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(int)); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - return rc; - } - - /* see if they want us to do the handshake */ - if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { - if (NULL == pmix_globals.mypeer->compat.psec->client_handshake) { - return PMIX_ERR_HANDSHAKE_FAILED; - } - if (PMIX_SUCCESS != (rc = pmix_globals.mypeer->compat.psec->client_handshake(sd))) { - return rc; - } - } else if (PMIX_SUCCESS != reply) { - return reply; - } - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: RECV CONNECT CONFIRMATION"); - - /* receive our index into the server's client array */ - rc = pmix_usock_recv_blocking(sd, (char*)&pmix_globals.pindex, sizeof(int)); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - return rc; - } - if (sockopt) { - /* return the socket to normal */ - if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) { - return PMIX_ERR_UNREACH; - } - } - - return PMIX_SUCCESS; -} - -static pmix_status_t usock_connect(struct sockaddr *addr, int *fd) -{ - int sd=-1; - pmix_status_t rc; - pmix_socklen_t addrlen = 0; - int retries = 0; - - pmix_output_verbose(2, pmix_globals.debug_output, - "usock_peer_try_connect: attempting to connect to server"); - - addrlen = sizeof(struct sockaddr_un); - while (retries < PMIX_MAX_RETRIES) { - retries++; - /* Create the new socket */ - sd = socket(PF_UNIX, SOCK_STREAM, 0); - if (sd < 0) { - pmix_output(0, "pmix:create_socket: socket() failed: %s (%d)\n", - strerror(pmix_socket_errno), - pmix_socket_errno); - continue; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "usock_peer_try_connect: attempting to connect to server on socket %d", sd); - /* try to connect */ - if (connect(sd, addr, addrlen) < 0) { - if (pmix_socket_errno == ETIMEDOUT) { - /* The server may be too busy to accept new connections */ - pmix_output_verbose(2, pmix_globals.debug_output, - "timeout connecting to server"); - CLOSE_THE_SOCKET(sd); - continue; - } - - /* Some kernels (Linux 2.6) will automatically software - abort a connection that was ECONNREFUSED on the last - attempt, without even trying to establish the - connection. Handle that case in a semi-rational - way by trying twice before giving up */ - if (ECONNABORTED == pmix_socket_errno) { - pmix_output_verbose(2, pmix_globals.debug_output, - "connection to server aborted by OS - retrying"); - CLOSE_THE_SOCKET(sd); - continue; - } else { - pmix_output_verbose(2, pmix_globals.debug_output, - "Connect failed: %s (%d)", strerror(pmix_socket_errno), - pmix_socket_errno); - CLOSE_THE_SOCKET(sd); - continue; - } - } else { - /* otherwise, the connect succeeded - so break out of the loop */ - break; - } - } - - if (retries == PMIX_MAX_RETRIES || sd < 0){ - /* We were unsuccessful in establishing this connection, and are - * not likely to suddenly become successful */ - if (0 <= sd) { - CLOSE_THE_SOCKET(sd); - } - return PMIX_ERR_UNREACH; - } - - /* send our identity and any authentication credentials to the server */ - if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) { - CLOSE_THE_SOCKET(sd); - return rc; - } - - /* do whatever handshake is required */ - if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) { - CLOSE_THE_SOCKET(sd); - return rc; - } - - pmix_output_verbose(2, pmix_globals.debug_output, - "sock_peer_try_connect: Connection across to server succeeded"); - - /* mark the connection as made */ - pmix_globals.connected = true; - - pmix_usock_set_nonblocking(sd); - - *fd = sd; - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c index 18322f9e42..1519191e27 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c @@ -51,13 +51,14 @@ #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/output.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #include "pmix_client_ops.h" #include "src/include/pmix_jobdata.h" /* callback for wait completion */ -static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); static void op_cbfunc(pmix_status_t status, void *cbdata); @@ -167,9 +168,12 @@ PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t npro cb->cbdata = cbdata; /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){ + PMIX_RELEASE(msg); + PMIX_RELEASE(cb); + } - return PMIX_SUCCESS; + return rc; } PMIX_EXPORT pmix_status_t PMIx_Disconnect(const pmix_proc_t procs[], size_t nprocs, @@ -274,15 +278,19 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t procs[], size_t n cb->cbdata = cbdata; /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){ + PMIX_RELEASE(msg); + PMIX_RELEASE(cb); + } pmix_output_verbose(2, pmix_globals.debug_output, "pmix: disconnect completed"); - return PMIX_SUCCESS; + return rc; } -static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_cb_t *cb = (pmix_cb_t*)cbdata; diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c index a6655c67bc..644931bda5 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c @@ -52,7 +52,7 @@ #include "src/util/error.h" #include "src/util/hash.h" #include "src/util/output.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #include "pmix_client_ops.h" @@ -61,7 +61,7 @@ static pmix_status_t pack_fence(pmix_buffer_t *msg, pmix_cmd_t cmd, const pmix_proc_t *procs, size_t nprocs, const pmix_info_t *info, size_t ninfo); static void wait_cbfunc(struct pmix_peer_t *pr, - pmix_usock_hdr_t *hdr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); static void op_cbfunc(pmix_status_t status, void *cbdata); @@ -160,9 +160,11 @@ PMIX_EXPORT pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs cb->cbdata = cbdata; /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); - - return PMIX_SUCCESS; + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){ + PMIX_RELEASE(msg); + PMIX_RELEASE(cb); + } + return rc; } static pmix_status_t unpack_return(pmix_buffer_t *data) @@ -223,7 +225,7 @@ static pmix_status_t pack_fence(pmix_buffer_t *msg, pmix_cmd_t cmd, return PMIX_SUCCESS; } -static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_cbfunc(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_cb_t *cb = (pmix_cb_t*)cbdata; diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c index 256b5ee1b4..7e5dce6ece 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c @@ -53,7 +53,7 @@ #include "src/util/error.h" #include "src/util/hash.h" #include "src/util/output.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) #include "src/dstore/pmix_dstore.h" #endif /* PMIX_ENABLE_DSTORE */ @@ -67,8 +67,9 @@ static pmix_buffer_t* _pack_get(char *nspace, pmix_rank_t rank, static void _getnbfn(int sd, short args, void *cbdata); -static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata); +static void _getnb_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata); static void _value_cbfunc(pmix_status_t status, pmix_value_t *kv, void *cbdata); @@ -234,7 +235,8 @@ static pmix_buffer_t* _pack_get(char *nspace, pmix_rank_t rank, /* this callback is coming from the usock recv, and thus * is occurring inside of our progress thread - hence, no * need to thread shift */ -static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void _getnb_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_cb_t *cb = (pmix_cb_t*)cbdata; @@ -737,10 +739,13 @@ static void _getnbfn(int fd, short flags, void *cbdata) pmix_globals.myid.nspace, pmix_globals.myid.rank, cb->nspace, cb->rank, cb->key); - /* create a callback object as we need to pass it to the - * recv routine so we know which callback to use when - * the return message is recvd */ + /* track the callback object */ pmix_list_append(&pmix_client_globals.pending_requests, &cb->super); - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, _getnb_cbfunc, cb); + /* send to the server */ + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, _getnb_cbfunc, (void*)cb))){ + pmix_list_remove_item(&pmix_client_globals.pending_requests, &cb->super); + cb->value_cbfunc(PMIX_ERROR, NULL, cb->cbdata); + PMIX_RELEASE(cb); + return; + } } diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h index 640a76cda3..bedb6fcaed 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h @@ -15,7 +15,6 @@ #include "src/buffer_ops/buffer_ops.h" #include "src/class/pmix_hash_table.h" -#include "src/usock/usock.h" BEGIN_C_DECLS diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c index a394965c90..59b1610012 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c @@ -51,14 +51,16 @@ #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/output.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #include "pmix_client_ops.h" -static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); static void op_cbfunc(pmix_status_t status, void *cbdata); -static void wait_lookup_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_lookup_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); static void lookup_cbfunc(pmix_status_t status, pmix_pdata_t pdata[], size_t ndata, void *cbdata); @@ -163,13 +165,16 @@ PMIX_EXPORT pmix_status_t PMIx_Publish_nb(const pmix_info_t info[], size_t ninfo cb->active = true; /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){ + PMIX_RELEASE(msg); + PMIX_RELEASE(cb); + } - return PMIX_SUCCESS; + return rc; } PMIX_EXPORT pmix_status_t PMIx_Lookup(pmix_pdata_t pdata[], size_t ndata, - const pmix_info_t info[], size_t ninfo) + const pmix_info_t info[], size_t ninfo) { pmix_status_t rc; pmix_cb_t *cb; @@ -289,10 +294,13 @@ PMIX_EXPORT pmix_status_t PMIx_Lookup_nb(char **keys, cb->lookup_cbfunc = cbfunc; cb->cbdata = cbdata; - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_lookup_cbfunc, cb); + /* send to the server */ + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_lookup_cbfunc, (void*)cb))){ + PMIX_RELEASE(msg); + PMIX_RELEASE(cb); + } - return PMIX_SUCCESS; + return rc; } PMIX_EXPORT pmix_status_t PMIx_Unpublish(char **keys, @@ -391,13 +399,17 @@ PMIX_EXPORT pmix_status_t PMIx_Unpublish_nb(char **keys, cb->cbdata = cbdata; cb->active = true; - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); + /* send to the server */ + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){ + PMIX_RELEASE(msg); + PMIX_RELEASE(cb); + } - return PMIX_SUCCESS; + return rc; } -static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_cb_t *cb = (pmix_cb_t*)cbdata; @@ -428,7 +440,8 @@ static void op_cbfunc(pmix_status_t status, void *cbdata) cb->active = false; } -static void wait_lookup_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_lookup_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_cb_t *cb = (pmix_cb_t*)cbdata; diff --git a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c index 728ae47363..48188c872a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c +++ b/opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c @@ -51,12 +51,13 @@ #include "src/util/argv.h" #include "src/util/error.h" #include "src/util/output.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #include "pmix_client_ops.h" #include "src/include/pmix_jobdata.h" -static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); static void spawn_cbfunc(pmix_status_t status, char nspace[], void *cbdata); @@ -169,13 +170,17 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin cb->cbdata = cbdata; /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb); + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){ + PMIX_RELEASE(msg); + PMIX_RELEASE(cb); + } - return PMIX_SUCCESS; + return rc; } /* callback for wait completion */ -static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void wait_cbfunc(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_cb_t *cb = (pmix_cb_t*)cbdata; diff --git a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c index b8060bcc1b..6fb39262a7 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c +++ b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c @@ -25,14 +25,14 @@ #include "src/util/error.h" #include "src/util/output.h" #include "src/buffer_ops/buffer_ops.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #include "src/client/pmix_client_ops.h" #include "src/server/pmix_server_ops.h" #include "src/include/pmix_globals.h" static void log_cbfunc(struct pmix_peer_t *peer, - pmix_usock_hdr_t *hdr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata; @@ -84,6 +84,7 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, pmix_host_server.log(&pmix_globals.myid, data, ndata, directives, ndirs, cbfunc, cbdata); + rc = PMIX_SUCCESS; } else { /* if we are a client, then relay this request to the server */ cd = PMIX_NEW(pmix_shift_caddy_t); @@ -125,7 +126,9 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata, pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query sending to server"); - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, log_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, log_cbfunc, (void*)cd))){ + PMIX_RELEASE(cd); + } } - return PMIX_SUCCESS; + return rc; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c index 672330a201..77f86755b8 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c +++ b/opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c @@ -25,7 +25,7 @@ #include "src/util/error.h" #include "src/util/output.h" #include "src/buffer_ops/buffer_ops.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #include "src/client/pmix_client_ops.h" #include "src/server/pmix_server_ops.h" @@ -44,7 +44,7 @@ static void relcbfunc(void *cbdata) PMIX_RELEASE(cd); } static void query_cbfunc(struct pmix_peer_t *peer, - pmix_usock_hdr_t *hdr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata; @@ -124,6 +124,7 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque pmix_host_server.query(&pmix_globals.myid, queries, nqueries, cbfunc, cbdata); + rc = PMIX_SUCCESS; } else { /* if we are a client, then relay this request to the server */ cd = PMIX_NEW(pmix_query_caddy_t); @@ -150,7 +151,9 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque } pmix_output_verbose(2, pmix_globals.debug_output, "pmix:query sending to server"); - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, query_cbfunc, cd); + if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, query_cbfunc, (void*)cd))){ + PMIX_RELEASE(cd); + } } - return PMIX_SUCCESS; + return rc; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c index 806e583c8c..d3ffd47f1a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c +++ b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c @@ -62,7 +62,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, return rc; } -static void notify_event_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void notify_event_cbfunc(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_status_t rc, ret; @@ -155,8 +155,13 @@ static pmix_status_t notify_server_of_event(pmix_status_t status, cb = PMIX_NEW(pmix_cb_t); cb->op_cbfunc = cbfunc; cb->cbdata = cbdata; - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, notify_event_cbfunc, cb); + /* send to the server */ + rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, notify_event_cbfunc, cb); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(cb); + goto cleanup; + } /* now notify any matching registered callbacks we have */ pmix_invoke_local_event_hdlr(chain); diff --git a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c index 3b751b5c22..437106187b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c +++ b/opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c @@ -59,7 +59,7 @@ PMIX_CLASS_INSTANCE(pmix_rshift_caddy_t, rscon, rsdes); -static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, +static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_rshift_caddy_t *rb = (pmix_rshift_caddy_t*)cbdata; @@ -152,9 +152,13 @@ static pmix_status_t _send_to_server(pmix_rshift_caddy_t *rcd) return rc; } } - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, regevents_cbfunc, rcd); + rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, regevents_cbfunc, rcd); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(msg); + } - return PMIX_SUCCESS; + return rc; } static pmix_status_t _add_hdlr(pmix_list_t *list, pmix_list_item_t *item, @@ -565,8 +569,11 @@ static void dereg_event_hdlr(int sd, short args, void *cbdata) report: if (NULL != msg) { - /* push the message into our event base to send to the server */ - PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, NULL, NULL); + /* send to the server */ + rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, NULL, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } } cleanup: diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c index 2014b8546e..a6f9d6c85d 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c @@ -40,26 +40,75 @@ #include "src/class/pmix_hash_table.h" #include "src/class/pmix_list.h" - -void pmix_globals_init(void) +static void cbcon(pmix_cb_t *p) { - memset(&pmix_globals.myid, 0, sizeof(pmix_proc_t)); - PMIX_CONSTRUCT(&pmix_globals.nspaces, pmix_list_t); - PMIX_CONSTRUCT(&pmix_globals.events, pmix_events_t); + p->active = false; + p->checked = false; + PMIX_CONSTRUCT(&p->data, pmix_buffer_t); + p->cbfunc = NULL; + p->op_cbfunc = NULL; + p->value_cbfunc = NULL; + p->lookup_cbfunc = NULL; + p->spawn_cbfunc = NULL; + p->cbdata = NULL; + memset(p->nspace, 0, PMIX_MAX_NSLEN+1); + p->rank = -1; + p->key = NULL; + p->value = NULL; + p->procs = NULL; + p->info = NULL; + p->ninfo = 0; + p->nvals = 0; } - -void pmix_globals_finalize(void) +static void cbdes(pmix_cb_t *p) { - PMIX_LIST_DESTRUCT(&pmix_globals.nspaces); - if (NULL != pmix_globals.cache_local) { - PMIX_RELEASE(pmix_globals.cache_local); - } - if (NULL != pmix_globals.cache_remote) { - PMIX_RELEASE(pmix_globals.cache_remote); - } - PMIX_DESTRUCT(&pmix_globals.events); + PMIX_DESTRUCT(&p->data); } +PMIX_CLASS_INSTANCE(pmix_cb_t, + pmix_list_item_t, + cbcon, cbdes); +static void pcon(pmix_peer_t *p) +{ + p->info = NULL; + p->proc_cnt = 0; + p->server_object = NULL; + p->index = 0; + p->sd = -1; + p->send_ev_active = false; + p->recv_ev_active = false; + PMIX_CONSTRUCT(&p->send_queue, pmix_list_t); + p->send_msg = NULL; + p->recv_msg = NULL; + memset(&p->compat, 0, sizeof(p->compat)); +} +static void pdes(pmix_peer_t *p) +{ + if (0 <= p->sd) { + CLOSE_THE_SOCKET(p->sd); + } + if (p->send_ev_active) { + event_del(&p->send_event); + } + if (p->recv_ev_active) { + event_del(&p->recv_event); + } + + if (NULL != p->info) { + PMIX_RELEASE(p->info); + } + + PMIX_LIST_DESTRUCT(&p->send_queue); + if (NULL != p->send_msg) { + PMIX_RELEASE(p->send_msg); + } + if (NULL != p->recv_msg) { + PMIX_RELEASE(p->recv_msg); + } +} +PMIX_CLASS_INSTANCE(pmix_peer_t, + pmix_object_t, + pcon, pdes); static void nscon(pmix_nspace_t *p) { diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h index 2e40b75f89..9709f4a880 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h @@ -38,6 +38,7 @@ #include "src/event/pmix_event.h" #include "src/mca/psec/psec.h" +#include "src/mca/ptl/ptl.h" BEGIN_C_DECLS @@ -92,14 +93,11 @@ typedef enum { PMIX_PROC_TOOL } pmix_proc_type_t; +/* defins some convenience macros for testing proc type */ +#define PMIX_PROC_IS_CLIENT (PMIX_PROC_CLIENT == pmix_globals.proc_type) +#define PMIX_PROC_IS_SERVER (PMIX_PROC_SERVER == pmix_globals.proc_type) +#define PMIX_PROC_IS_TOOL (PMIX_PROC_TOOL == pmix_globals.proc_type) -/**** MESSAGING STRUCTURES ****/ -/* header for messages */ -typedef struct { - int pindex; - uint32_t tag; - size_t nbytes; -} pmix_usock_hdr_t; /* internally used object for transferring data * to/from the server and for storing in the @@ -114,37 +112,6 @@ PMIX_CLASS_DECLARATION(pmix_kval_t); // forward declaration struct pmix_peer_t; -/* internally used cbfunc */ -typedef void (*pmix_usock_cbfunc_t)(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, - pmix_buffer_t *buf, void *cbdata); - -/* usock structure for sending a message */ -typedef struct { - pmix_list_item_t super; - pmix_event_t ev; - pmix_usock_hdr_t hdr; - pmix_buffer_t *data; - bool hdr_sent; - char *sdptr; - size_t sdbytes; -} pmix_usock_send_t; -PMIX_CLASS_DECLARATION(pmix_usock_send_t); - -/* usock structure for recving a message */ -typedef struct { - pmix_list_item_t super; - pmix_event_t ev; - struct pmix_peer_t *peer; - int sd; - pmix_usock_hdr_t hdr; - char *data; - bool hdr_recvd; - char *rdptr; - size_t rdbytes; -} pmix_usock_recv_t; -PMIX_CLASS_DECLARATION(pmix_usock_recv_t); - - /**** PEER STRUCTURES ****/ /* objects for tracking active nspaces */ typedef struct { @@ -185,6 +152,7 @@ PMIX_CLASS_DECLARATION(pmix_rank_info_t); * to plugins for cross-version support */ typedef struct pmix_personality_t { pmix_psec_module_t *psec; + pmix_ptl_module_t *ptl; } pmix_personality_t; /* object for tracking peers - each peer can have multiple @@ -199,13 +167,13 @@ typedef struct pmix_peer_t { void *server_object; int index; int sd; - pmix_event_t send_event; /**< registration with event thread for send events */ + pmix_event_t send_event; /**< registration with event thread for send events */ bool send_ev_active; - pmix_event_t recv_event; /**< registration with event thread for recv events */ + pmix_event_t recv_event; /**< registration with event thread for recv events */ bool recv_ev_active; - pmix_list_t send_queue; /**< list of messages to send */ - pmix_usock_send_t *send_msg; /**< current send in progress */ - pmix_usock_recv_t *recv_msg; /**< current recv in progress */ + pmix_list_t send_queue; /**< list of messages to send */ + pmix_ptl_send_t *send_msg; /**< current send in progress */ + pmix_ptl_recv_t *recv_msg; /**< current recv in progress */ pmix_personality_t compat; } pmix_peer_t; PMIX_CLASS_DECLARATION(pmix_peer_t); @@ -230,7 +198,7 @@ PMIX_CLASS_DECLARATION(pmix_snd_caddy_t); * request into the server's event base */ typedef struct { pmix_list_item_t super; - pmix_usock_hdr_t hdr; + pmix_ptl_hdr_t hdr; pmix_peer_t *peer; pmix_snd_caddy_t snd; } pmix_server_caddy_t; @@ -323,6 +291,35 @@ PMIX_CLASS_DECLARATION(pmix_job_data_caddy_t); } pmix_shift_caddy_t; PMIX_CLASS_DECLARATION(pmix_shift_caddy_t); +/* struct for tracking ops */ +typedef struct { + pmix_list_item_t super; + pmix_event_t ev; + volatile bool active; + bool checked; + int status; + pmix_status_t pstatus; + pmix_scope_t scope; + pmix_buffer_t data; + pmix_ptl_cbfunc_t cbfunc; + pmix_op_cbfunc_t op_cbfunc; + pmix_value_cbfunc_t value_cbfunc; + pmix_lookup_cbfunc_t lookup_cbfunc; + pmix_spawn_cbfunc_t spawn_cbfunc; + pmix_evhdlr_reg_cbfunc_t errreg_cbfunc; + size_t errhandler_ref; + void *cbdata; + char nspace[PMIX_MAX_NSLEN+1]; + pmix_rank_t rank; + char *key; + pmix_value_t *value; + pmix_proc_t *procs; + pmix_info_t *info; + size_t ninfo; + size_t nvals; +} pmix_cb_t; +PMIX_CLASS_DECLARATION(pmix_cb_t); + /* define a very simple caddy for dealing with pmix_info_t * objects when transferring portions of arrays */ typedef struct { @@ -371,12 +368,6 @@ typedef struct { } pmix_globals_t; -/* initialize the pmix_global structure */ -void pmix_globals_init(void); - -/* finalize the pmix_global structure */ -void pmix_globals_finalize(void); - extern pmix_globals_t pmix_globals; END_C_DECLS diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c index a54613d231..03ae705735 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c @@ -1926,13 +1926,13 @@ static char *source_name(pmix_mca_base_var_t *var) static int var_value_string (pmix_mca_base_var_t *var, char **value_string) { - const pmix_mca_base_var_storage_t *value; + const pmix_mca_base_var_storage_t *value=NULL; int ret; assert (PMIX_MCA_BASE_VAR_TYPE_MAX > var->mbv_type); ret = pmix_mca_base_var_get_value(var->mbv_index, &value, NULL, NULL); - if (PMIX_SUCCESS !=ret) { + if (PMIX_SUCCESS != ret || NULL == value) { return ret; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h index 30eebcd61b..29c75b869c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h @@ -89,7 +89,7 @@ BEGIN_C_DECLS typedef struct pmix_pif_t { pmix_list_item_t super; - char if_name[IF_NAMESIZE]; + char if_name[IF_NAMESIZE+1]; int if_index; uint16_t if_kernel_index; uint16_t af_family; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/base.h index a0872799ee..30ee996e66 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/base.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/base.h @@ -80,6 +80,18 @@ typedef struct pmix_psec_globals_t pmix_psec_globals_t; extern pmix_psec_globals_t pmix_psec_globals; +PMIX_EXPORT char* pmix_psec_base_get_available_modules(void); +PMIX_EXPORT pmix_status_t pmix_psec_base_assign_module(struct pmix_peer_t *peer, + const char *options); +PMIX_EXPORT pmix_status_t pmix_psec_base_create_cred(struct pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char **cred, size_t *len); +PMIX_EXPORT pmix_status_t pmix_psec_base_client_handshake(struct pmix_peer_t *peer, int sd); +PMIX_EXPORT pmix_status_t pmix_psec_base_validate_connection(struct pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char *cred, size_t len); + + END_C_DECLS #endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_fns.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_fns.c index 72ac7f26ab..93ad018501 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_fns.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_fns.c @@ -18,8 +18,9 @@ #include "src/class/pmix_list.h" #include "src/util/argv.h" +#include "src/util/error.h" #include "src/util/output.h" - +#include "src/mca/ptl/base/base.h" #include "src/mca/psec/base/base.h" @@ -43,16 +44,19 @@ char* pmix_psec_base_get_available_modules(void) return reply; } -pmix_psec_module_t* pmix_psec_base_assign_module(const char *options) +pmix_status_t pmix_psec_base_assign_module(struct pmix_peer_t *peer, + const char *options) { + pmix_peer_t *pr = (pmix_peer_t*)peer; pmix_psec_base_active_module_t *active; + pmix_psec_module_t *mod; char **tmp=NULL; int i; - pmix_psec_module_t *mod; if (!pmix_psec_globals.initialized) { - return NULL; + return PMIX_ERR_INIT; } + if (NULL != options) { tmp = pmix_argv_split(options, ','); } @@ -60,14 +64,16 @@ pmix_psec_module_t* pmix_psec_base_assign_module(const char *options) PMIX_LIST_FOREACH(active, &pmix_psec_globals.actives, pmix_psec_base_active_module_t) { if (NULL == tmp) { if (NULL != (mod = active->component->assign_module())) { - return mod; + pr->compat.psec = mod; + return PMIX_SUCCESS; } } else { for (i=0; NULL != tmp[i]; i++) { if (0 == strcmp(tmp[i], active->component->base.pmix_mca_component_name)) { if (NULL != (mod = active->component->assign_module())) { pmix_argv_free(tmp); - return mod; + pr->compat.psec = mod; + return PMIX_SUCCESS; } } } @@ -78,5 +84,69 @@ pmix_psec_module_t* pmix_psec_base_assign_module(const char *options) if (NULL != tmp) { pmix_argv_free(tmp); } - return NULL; + return PMIX_ERR_NOT_AVAILABLE; +} + +pmix_status_t pmix_psec_base_create_cred(struct pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char **cred, size_t *len) +{ + pmix_peer_t *pr = (pmix_peer_t*)peer; + + if (NULL == pr->compat.psec->create_cred) { + return PMIX_ERR_NOT_SUPPORTED; + } + return pr->compat.psec->create_cred(protocol, cred, len); +} + +pmix_status_t pmix_psec_base_client_handshake(struct pmix_peer_t *peer, int sd) +{ + pmix_peer_t *pr = (pmix_peer_t*)peer; + + if (NULL == pr->compat.psec->client_handshake) { + return PMIX_ERR_NOT_SUPPORTED; + } + return pr->compat.psec->client_handshake(sd); +} + +pmix_status_t pmix_psec_base_validate_connection(struct pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char *cred, size_t len) +{ + pmix_peer_t *pr = (pmix_peer_t*)peer; + pmix_status_t rc; + + /* if a credential is available, then check it */ + if (NULL != pr->compat.psec->validate_cred) { + if (PMIX_SUCCESS != (rc = pr->compat.psec->validate_cred(peer, protocol, cred, len))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "validation of credential failed: %s", + PMIx_Error_string(rc)); + return rc; + } + pmix_output_verbose(2, pmix_globals.debug_output, + "credential validated"); + /* send them success */ + rc = PMIX_SUCCESS; + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pr->sd, (char*)&rc, sizeof(int)))) { + PMIX_ERROR_LOG(rc); + } + return rc; + } else if (NULL != pr->compat.psec->server_handshake) { + /* execute the handshake if the security mode calls for it */ + pmix_output_verbose(2, pmix_globals.debug_output, + "executing handshake"); + rc = PMIX_ERR_READY_FOR_HANDSHAKE; + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pr->sd, (char*)&rc, sizeof(int)))) { + PMIX_ERROR_LOG(rc); + return rc; + } + if (PMIX_SUCCESS != (rc = pr->compat.psec->server_handshake(peer))) { + PMIX_ERROR_LOG(rc); + } + return rc; + } else { + /* this is not allowed */ + return PMIX_ERR_NOT_SUPPORTED; + } } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_frame.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_frame.c index b3a899bdc3..a0ae2e098a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_frame.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_frame.c @@ -45,6 +45,13 @@ /* Instantiate the global vars */ pmix_psec_globals_t pmix_psec_globals = {{{0}}}; +pmix_psec_API_t pmix_psec = { + .get_available_modules = pmix_psec_base_get_available_modules, + .assign_module = pmix_psec_base_assign_module, + .create_cred = pmix_psec_base_create_cred, + .client_handshake = pmix_psec_base_client_handshake, + .validate_connection = pmix_psec_base_validate_connection +}; static pmix_status_t pmix_psec_close(void) { diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c index 9beb46940d..fcf7834aab 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c @@ -28,10 +28,12 @@ #include "src/mca/psec/psec.h" #include "psec_munge.h" -static int munge_init(void); +static pmix_status_t munge_init(void); static void munge_finalize(void); -static char* create_cred(void); -static int validate_cred(pmix_peer_t *peer, char *cred); +static pmix_status_t create_cred(pmix_listener_protocol_t protocol, + char **cred, size_t *len); +static pmix_status_t validate_cred(pmix_listener_protocol_t protocol, + pmix_peer_t *peer, char *cred, size_t len); pmix_psec_module_t pmix_munge_module = { "munge", @@ -47,7 +49,7 @@ static char *mycred = NULL; static bool initialized = false; static bool refresh = false; -static int munge_init(void) +static pmix_status_t munge_init(void) { int rc; @@ -81,10 +83,10 @@ static void munge_finalize(void) } } -static char* create_cred(void) +static pmix_status_t create_cred(pmix_listener_protocol_t protocol, + char **cred, size_t *len) { int rc; - char *resp=NULL; pmix_output_verbose(2, pmix_globals.debug_output, "psec: munge create_cred"); @@ -92,7 +94,8 @@ static char* create_cred(void) if (initialized) { if (!refresh) { refresh = true; - resp = strdup(mycred); + *cred = strdup(mycred); + *len = strlen(mycred) + 1; } else { /* munge does not allow reuse of a credential, so we have to * refresh it for every use */ @@ -105,13 +108,15 @@ static char* create_cred(void) munge_strerror(rc)); return NULL; } - resp = strdup(mycred); + *cred = strdup(mycred); + *len = strlen(mycred) + 1; } } - return resp; + return PMIX_SUCCESS; } -static int validate_cred(pmix_peer_t *peer, char *cred) +static pmix_status_t validate_cred(pmix_listener_protocol_t protocol, + pmix_peer_t *peer, char *cred, size_t len) { uid_t uid; gid_t gid; diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c index d9a214cfb5..b6bb634ba3 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c @@ -11,6 +11,11 @@ #include +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif + #include #include "src/include/pmix_socket_errno.h" @@ -19,26 +24,26 @@ #include "src/util/error.h" #include "src/util/output.h" -#include -#ifdef HAVE_SYS_TYPES_H -#include -#endif - #include "src/mca/psec/psec.h" #include "psec_native.h" -static int native_init(void); +static pmix_status_t native_init(void); static void native_finalize(void); -static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred); +static pmix_status_t create_cred(pmix_listener_protocol_t protocol, + char **cred, size_t *len); +static pmix_status_t validate_cred(pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char *cred, size_t len); pmix_psec_module_t pmix_native_module = { .name = "native", .init = native_init, .finalize = native_finalize, + .create_cred = create_cred, .validate_cred = validate_cred }; -static int native_init(void) +static pmix_status_t native_init(void) { pmix_output_verbose(2, pmix_globals.debug_output, "psec: native init"); @@ -51,7 +56,44 @@ static void native_finalize(void) "psec: native finalize"); } -static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred) +static pmix_status_t create_cred(pmix_listener_protocol_t protocol, + char **cred, size_t *len) +{ + uid_t euid; + gid_t egid; + char *tmp, *ptr; + + if (PMIX_PROTOCOL_V1 == protocol || + PMIX_PROTOCOL_V3 == protocol) { + /* these are usock protocols - nothing to do */ + *cred = NULL; + *len = 0; + return PMIX_SUCCESS; + } + if (PMIX_PROTOCOL_V2 == protocol) { + /* tcp protocol - need to provide our effective + * uid and gid for validation on remote end */ + tmp = (char*)malloc(sizeof(uid_t) + sizeof(gid_t)); + if (NULL == tmp) { + return PMIX_ERR_NOMEM; + } + euid = geteuid(); + memcpy(tmp, &euid, sizeof(uid_t)); + ptr = tmp + sizeof(uid_t); + egid = getegid(); + memcpy(ptr, &egid, sizeof(gid_t)); + *cred = tmp; + *len = sizeof(uid_t) + sizeof(gid_t); + return PMIX_SUCCESS; + } + + /* unrecognized protocol */ + return PMIX_ERR_NOT_SUPPORTED; +} + +static pmix_status_t validate_cred(pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char *cred, size_t len) { #if defined(SO_PEERCRED) #ifdef HAVE_STRUCT_SOCKPEERCRED_UID @@ -64,62 +106,104 @@ static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred) #endif uid_t euid; gid_t gid; + char *ptr; + size_t ln; pmix_output_verbose(2, pmix_globals.debug_output, "psec: native validate_cred %s", cred ? cred : "NULL"); + if (PMIX_PROTOCOL_V1 == protocol || + PMIX_PROTOCOL_V3 == protocol) { + /* these are usock protocols - get the remote side's uid/gid */ #if defined(SO_PEERCRED) && (defined(HAVE_STRUCT_UCRED_UID) || defined(HAVE_STRUCT_UCRED_CR_UID)) - /* Ignore received 'cred' and validate ucred for socket instead. */ - pmix_output_verbose(2, pmix_globals.debug_output, - "psec:native checking getsockopt for peer credentials"); - if (getsockopt (peer->sd, SOL_SOCKET, SO_PEERCRED, &ucred, &crlen) < 0) { + /* Ignore received 'cred' and validate ucred for socket instead. */ pmix_output_verbose(2, pmix_globals.debug_output, - "psec: getsockopt SO_PEERCRED failed: %s", - strerror (pmix_socket_errno)); - pmix_output(0, "ONE"); - return PMIX_ERR_INVALID_CRED; - } + "psec:native checking getsockopt on socket %d for peer credentials", peer->sd); + if (getsockopt (peer->sd, SOL_SOCKET, SO_PEERCRED, &ucred, &crlen) < 0) { + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: getsockopt SO_PEERCRED failed: %s", + strerror (pmix_socket_errno)); + return PMIX_ERR_INVALID_CRED; + } #if defined(HAVE_STRUCT_UCRED_UID) - euid = ucred.uid; - gid = ucred.gid; + euid = ucred.uid; + gid = ucred.gid; #else - euid = ucred.cr_uid; - gid = ucred.cr_gid; + euid = ucred.cr_uid; + gid = ucred.cr_gid; #endif #elif defined(HAVE_GETPEEREID) - pmix_output_verbose(2, pmix_globals.debug_output, - "psec:native checking getpeereid for peer credentials"); - if (0 != getpeereid(peer->sd, &euid, &gid)) { pmix_output_verbose(2, pmix_globals.debug_output, - "psec: getsockopt getpeereid failed: %s", - strerror (pmix_socket_errno)); - pmix_output(0, "TWO"); - return PMIX_ERR_INVALID_CRED; + "psec:native checking getpeereid on socket %d for peer credentials", peer->sd); + if (0 != getpeereid(peer->sd, &euid, &gid)) { + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: getsockopt getpeereid failed: %s", + strerror (pmix_socket_errno)); + return PMIX_ERR_INVALID_CRED; } #else - pmix_output(0, "FIVE"); - return PMIX_ERR_NOT_SUPPORTED; + return PMIX_ERR_NOT_SUPPORTED; #endif - /* check uid */ - if (euid != peer->info->uid) { + /* check uid */ + if (euid != peer->info->uid) { + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: socket cred contains invalid uid %u", euid); + return PMIX_ERR_INVALID_CRED; + } + + /* check gid */ + if (gid != peer->info->gid) { + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: socket cred contains invalid gid %u", gid); + return PMIX_ERR_INVALID_CRED; + } + pmix_output_verbose(2, pmix_globals.debug_output, - "psec: socket cred contains invalid uid %u", euid); - pmix_output(0, "THREE"); - return PMIX_ERR_INVALID_CRED; + "psec: native credential %u:%u valid", + euid, gid); + return PMIX_SUCCESS; } - /* check gid */ - if (gid != peer->info->gid) { + if (PMIX_PROTOCOL_V2 == protocol) { + /* this is a tcp protocol, so the cred is actually the uid/gid + * passed upwards from the client */ + ln = len; + euid = 0; + gid = 0; + if (sizeof(uid_t) <= ln) { + memcpy(&euid, cred, sizeof(uid_t)); + ln -= sizeof(uid_t); + ptr = cred + sizeof(uid_t); + } else { + return PMIX_ERR_INVALID_CRED; + } + if (sizeof(gid_t) <= ln) { + memcpy(&gid, ptr, sizeof(gid_t)); + } else { + return PMIX_ERR_INVALID_CRED; + } + /* check uid */ + if (euid != peer->info->uid) { + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: socket cred contains invalid uid %u", euid); + return PMIX_ERR_INVALID_CRED; + } + + /* check gid */ + if (gid != peer->info->gid) { + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: socket cred contains invalid gid %u", gid); + return PMIX_ERR_INVALID_CRED; + } + pmix_output_verbose(2, pmix_globals.debug_output, - "psec: socket cred contains invalid gid %u", gid); - pmix_output(0, "FOUR"); - return PMIX_ERR_INVALID_CRED; + "psec: native credential %u:%u valid", + euid, gid); + return PMIX_SUCCESS; } - pmix_output_verbose(2, pmix_globals.debug_output, - "psec: native credential %u:%u valid", - euid, gid); - return PMIX_SUCCESS; + /* don't recognize the protocol */ + return PMIX_ERR_NOT_SUPPORTED; } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am new file mode 100644 index 0000000000..7423699637 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am @@ -0,0 +1,50 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers = psec_none.h +sources = \ + psec_none_component.c \ + psec_none.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_psec_none_DSO +lib = +lib_sources = +component = mca_psec_none.la +component_sources = $(headers) $(sources) +else +lib = libmca_psec_none.la +lib_sources = $(headers) $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component) +mca_psec_none_la_SOURCES = $(component_sources) +mca_psec_none_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(lib) +libmca_psec_none_la_SOURCES = $(lib_sources) +libmca_psec_none_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c new file mode 100644 index 0000000000..5fc22cec04 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include + +#include "src/include/pmix_socket_errno.h" +#include "src/include/pmix_globals.h" +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/output.h" + +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#include "src/mca/psec/psec.h" +#include "psec_none.h" + +static pmix_status_t none_init(void); +static void none_finalize(void); +static pmix_status_t validate_cred(pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char *cred, size_t len); + +pmix_psec_module_t pmix_none_module = { + .name = "none", + .init = none_init, + .finalize = none_finalize, + .validate_cred = validate_cred +}; + +static pmix_status_t none_init(void) +{ + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: none init"); + return PMIX_SUCCESS; +} + +static void none_finalize(void) +{ + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: none finalize"); +} + +static pmix_status_t validate_cred(pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char *cred, size_t len) +{ + pmix_output_verbose(2, pmix_globals.debug_output, + "psec: none always reports valid"); + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.h b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.h new file mode 100644 index 0000000000..d443c97448 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_NATIVE_H +#define PMIX_NATIVE_H + +#include + + +#include "src/mca/psec/psec.h" + +BEGIN_C_DECLS + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_psec_base_component_t mca_psec_none_component; +extern pmix_psec_module_t pmix_none_module; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c new file mode 100644 index 0000000000..17e9035b3a --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c @@ -0,0 +1,91 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include +#include "pmix_common.h" + + +#include "src/mca/psec/psec.h" +#include "psec_none.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); +static pmix_psec_module_t* assign_module(void); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +pmix_psec_base_component_t mca_psec_none_component = { + .base = { + PMIX_PSEC_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "none", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + .assign_module = assign_module +}; + + +static int component_open(void) +{ + return PMIX_SUCCESS; +} + + +static int component_query(pmix_mca_base_module_t **module, int *priority) +{ + *priority = 0; + *module = (pmix_mca_base_module_t *)&pmix_none_module; + return PMIX_SUCCESS; +} + + +static int component_close(void) +{ + return PMIX_SUCCESS; +} + +static pmix_psec_module_t* assign_module(void) +{ + return &pmix_none_module; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h index 4c51efd452..c0cff0b091 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h @@ -33,6 +33,7 @@ #include "src/mca/mca.h" #include "src/mca/base/pmix_mca_base_var.h" #include "src/mca/base/pmix_mca_base_framework.h" +#include "src/mca/ptl/ptl_types.h" BEGIN_C_DECLS @@ -45,7 +46,7 @@ struct pmix_peer_t; * Initialize the module. Returns an error if the module cannot * run, success if it can and wants to be used. */ -typedef int (*pmix_psec_base_module_init_fn_t)(void); +typedef pmix_status_t (*pmix_psec_base_module_init_fn_t)(void); /** * Finalize the module. Tear down any allocated storage, disconnect @@ -55,10 +56,12 @@ typedef void (*pmix_psec_base_module_fini_fn_t)(void); /**** CLIENT-SIDE FUNCTIONS ****/ /** - * Create and return a string representation of a credential for this - * client + * Create and return a credential for this client - this + * could be a string or a byte array, which is why we must + * also return the length */ -typedef char* (*pmix_psec_base_module_create_cred_fn_t)(void); +typedef pmix_status_t (*pmix_psec_base_module_create_cred_fn_t)(pmix_listener_protocol_t protocol, + char **cred, size_t *len); /** * Perform the client-side handshake. Note that it is not required @@ -70,9 +73,12 @@ typedef pmix_status_t (*pmix_psec_base_module_client_hndshk_fn_t)(int sd); /**** SERVER-SIDE FUNCTIONS ****/ /** - * Validate a client's credential + * Validate a client's credential - the credential could be a string + * or an array of bytes, which is why we include the length */ -typedef pmix_status_t (*pmix_psec_base_module_validate_cred_fn_t)(struct pmix_peer_t *peer, char *cred); +typedef pmix_status_t (*pmix_psec_base_module_validate_cred_fn_t)(struct pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char *cred, size_t len); /** * Perform the server-side handshake. Note that it is not required @@ -98,6 +104,50 @@ typedef struct { } pmix_psec_module_t; +/* define an API module */ + +/* get a list of available options - caller must free results + * when done */ +typedef char* (*pmix_psec_API_get_available_modules_fn_t)(void); + +/* Select a psec module for a given peer */ +typedef pmix_status_t (*pmix_psec_API_assign_module_fn_t)(struct pmix_peer_t *peer, + const char *options); + +/** + * Create and return a string representation of a credential for this + * client + */ +typedef pmix_status_t (*pmix_psec_API_create_cred_fn_t)(struct pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char **cred, size_t *len); + +/** + * Perform the client-side handshake. Note that it is not required + * (and indeed, would be rare) for a protocol to use both the + * credential and handshake interfaces. It is acceptable, therefore, + * for one of them to be NULL */ +typedef pmix_status_t (*pmix_psec_API_client_hndshk_fn_t)(struct pmix_peer_t *peer, int sd); + + +/**** SERVER-SIDE FUNCTIONS ****/ +/** + * Validate a client's connection request + */ +typedef pmix_status_t (*pmix_psec_API_validate_connection_fn_t)(struct pmix_peer_t *peer, + pmix_listener_protocol_t protocol, + char *cred, size_t len); + +typedef struct { + pmix_psec_API_get_available_modules_fn_t get_available_modules; + pmix_psec_API_assign_module_fn_t assign_module; + pmix_psec_API_create_cred_fn_t create_cred; + pmix_psec_API_client_hndshk_fn_t client_handshake; + pmix_psec_API_validate_connection_fn_t validate_connection; +} pmix_psec_API_t; + +PMIX_EXPORT extern pmix_psec_API_t pmix_psec; + /**** COMPONENT STRUCTURE DEFINITION ****/ /* define a component-level API for initializing the component */ @@ -122,14 +172,6 @@ struct pmix_psec_base_component_t { }; typedef struct pmix_psec_base_component_t pmix_psec_base_component_t; -/* Select a psec module for a given peer */ -pmix_psec_module_t* pmix_psec_base_assign_module(const char *options); - -/* get a list of available options - caller must free results - * when done */ -char* pmix_psec_base_get_available_modules(void); - - /* * Macro for use in components that are of type psec */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/Makefile.am new file mode 100644 index 0000000000..dcc0b2691b --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/Makefile.am @@ -0,0 +1,44 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +AM_CPPFLAGS = $(LTDLINCL) + +# main library setup +noinst_LTLIBRARIES = libmca_ptl.la +libmca_ptl_la_SOURCES = + +# local files +headers = ptl.h ptl_types.h +sources = + +# Conditionally install the header files +if WANT_INSTALL_HEADERS +pmixdir = $(pmixincludedir)/$(subdir) +nobase_pmix_HEADERS = $(headers) +endif + +include base/Makefile.include + +libmca_ptl_la_SOURCES += $(headers) $(sources) + +distclean-local: + rm -f base/static-components.h diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/Makefile.include b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/Makefile.include new file mode 100644 index 0000000000..ef5342171a --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/Makefile.include @@ -0,0 +1,35 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# This makefile.include does not stand on its own - it is included from +# src/Makefile.am + +headers += \ + base/base.h + +sources += \ + base/ptl_base_frame.c \ + base/ptl_base_select.c \ + base/ptl_base_sendrecv.c \ + base/ptl_base_listener.c \ + base/ptl_base_stubs.c \ + base/ptl_base_connect.c diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h new file mode 100644 index 0000000000..6bf1eccb3e --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h @@ -0,0 +1,123 @@ +/* -*- C -*- + * + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ +#ifndef PMIX_PTL_BASE_H_ +#define PMIX_PTL_BASE_H_ + +#include + + +#ifdef HAVE_SYS_TIME_H +#include /* for struct timeval */ +#endif +#ifdef HAVE_STRING_H +#include +#endif +#if PMIX_HAVE_HWLOC +#include PMIX_HWLOC_HEADER +#endif + +#include "src/class/pmix_pointer_array.h" +#include "src/mca/mca.h" +#include "src/mca/base/pmix_mca_base_framework.h" + +#include "src/mca/ptl/ptl.h" + + + BEGIN_C_DECLS + +/* + * MCA Framework + */ +extern pmix_mca_base_framework_t pmix_ptl_base_framework; +/** + * PTL select function + * + * Cycle across available components and construct the list + * of active modules + */ +pmix_status_t pmix_ptl_base_select(void); + +/** + * Track an active component + */ +struct pmix_ptl_base_active_t { + pmix_list_item_t super; + pmix_status_t pri; + pmix_ptl_base_component_t *component; + pmix_ptl_module_t *module; +}; +typedef struct pmix_ptl_base_active_t pmix_ptl_base_active_t; +PMIX_CLASS_DECLARATION(pmix_ptl_base_active_t); + + +/* framework globals */ +struct pmix_ptl_globals_t { + pmix_list_t actives; + bool initialized; + pmix_list_t posted_recvs; // list of pmix_ptl_posted_recv_t + int stop_thread[2]; + bool listen_thread_active; + pmix_list_t listeners; +}; +typedef struct pmix_ptl_globals_t pmix_ptl_globals_t; + +extern pmix_ptl_globals_t pmix_ptl_globals; + +/* API stubs */ +PMIX_EXPORT pmix_status_t pmix_ptl_stub_set_notification_cbfunc(pmix_ptl_cbfunc_t cbfunc); +PMIX_EXPORT char* pmix_ptl_stub_get_available_modules(void); +PMIX_EXPORT pmix_status_t pmix_ptl_stub_send_recv(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_cbfunc_t cbfunc, + void *cbdata); +PMIX_EXPORT pmix_status_t pmix_ptl_stub_send_oneway(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_tag_t tag); +PMIX_EXPORT pmix_status_t pmix_ptl_stub_connect_to_peer(struct pmix_peer_t *peer, + pmix_info_t info[], size_t ninfo); + +PMIX_EXPORT pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo); +PMIX_EXPORT void pmix_ptl_base_stop_listening(void); + + +/* base support functions */ +PMIX_EXPORT void pmix_ptl_base_send(int sd, short args, void *cbdata); +PMIX_EXPORT void pmix_ptl_base_send_recv(int sd, short args, void *cbdata); +PMIX_EXPORT void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata); +PMIX_EXPORT void pmix_ptl_base_recv_handler(int sd, short flags, void *cbdata); +PMIX_EXPORT void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata); +PMIX_EXPORT pmix_status_t pmix_ptl_base_set_nonblocking(int sd); +PMIX_EXPORT pmix_status_t pmix_ptl_base_set_blocking(int sd); +PMIX_EXPORT pmix_status_t pmix_ptl_base_send_blocking(int sd, char *ptr, size_t size); +PMIX_EXPORT pmix_status_t pmix_ptl_base_recv_blocking(int sd, char *data, size_t size); +PMIX_EXPORT pmix_status_t pmix_ptl_base_connect(struct sockaddr_storage *addr, + pmix_socklen_t len, int *fd); +PMIX_EXPORT void pmix_ptl_base_connection_handler(int sd, short args, void *cbdata); +PMIX_EXPORT pmix_status_t pmix_ptl_base_send_connect_ack(int sd); +PMIX_EXPORT pmix_status_t pmix_ptl_base_recv_connect_ack(int sd); + + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_connect.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_connect.c new file mode 100644 index 0000000000..f66d61e641 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_connect.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include "include/pmix_stdint.h" + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif + +#include "include/pmix_socket_errno.h" +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/getid.h" +#include "src/util/strnlen.h" +#include "src/include/pmix_globals.h" +#include "src/client/pmix_client_ops.h" +#include "src/server/pmix_server_ops.h" + +#include "src/mca/ptl/base/base.h" + +pmix_status_t pmix_ptl_base_set_nonblocking(int sd) +{ + int flags; + /* setup the socket as non-blocking */ + if ((flags = fcntl(sd, F_GETFL, 0)) < 0) { + pmix_output(0, "ptl:base:set_nonblocking: fcntl(F_GETFL) failed: %s (%d)\n", + strerror(pmix_socket_errno), + pmix_socket_errno); + } else { + flags |= O_NONBLOCK; + if(fcntl(sd, F_SETFL, flags) < 0) + pmix_output(0, "ptl:base:set_nonblocking: fcntl(F_SETFL) failed: %s (%d)\n", + strerror(pmix_socket_errno), + pmix_socket_errno); + } + return PMIX_SUCCESS; +} + +pmix_status_t pmix_ptl_base_set_blocking(int sd) +{ + int flags; + /* setup the socket as non-blocking */ + if ((flags = fcntl(sd, F_GETFL, 0)) < 0) { + pmix_output(0, "ptl:base:set_blocking: fcntl(F_GETFL) failed: %s (%d)\n", + strerror(pmix_socket_errno), + pmix_socket_errno); + } else { + flags &= ~(O_NONBLOCK); + if(fcntl(sd, F_SETFL, flags) < 0) + pmix_output(0, "ptl:base:set_blocking: fcntl(F_SETFL) failed: %s (%d)\n", + strerror(pmix_socket_errno), + pmix_socket_errno); + } + return PMIX_SUCCESS; +} + +/* + * A blocking send on a non-blocking socket. Used to send the small amount of connection + * information that identifies the peers endpoint. + */ +pmix_status_t pmix_ptl_base_send_blocking(int sd, char *ptr, size_t size) +{ + size_t cnt = 0; + int retval; + + pmix_output_verbose(8, pmix_globals.debug_output, + "send blocking of %"PRIsize_t" bytes to socket %d", + size, sd ); + while (cnt < size) { + retval = send(sd, (char*)ptr+cnt, size-cnt, 0); + if (retval < 0) { + if (EAGAIN == pmix_socket_errno || + EWOULDBLOCK == pmix_socket_errno) { + /* just cycle and let it try again */ + pmix_output_verbose(8, pmix_globals.debug_output, + "blocking_send received error %d:%s from remote - cycling", + pmix_socket_errno, strerror(pmix_socket_errno)); + continue; + } + if (pmix_socket_errno != EINTR) { + pmix_output_verbose(8, pmix_globals.debug_output, + "ptl:base:peer_send_blocking: send() to socket %d failed: %s (%d)\n", + sd, strerror(pmix_socket_errno), + pmix_socket_errno); + return PMIX_ERR_UNREACH; + } + continue; + } + cnt += retval; + } + + pmix_output_verbose(8, pmix_globals.debug_output, + "blocking send complete to socket %d", sd); + return PMIX_SUCCESS; +} + +/* + * A blocking recv on a non-blocking socket. Used to receive the small amount of connection + * information that identifies the peers endpoint. + */ +pmix_status_t pmix_ptl_base_recv_blocking(int sd, char *data, size_t size) +{ + size_t cnt = 0; + + pmix_output_verbose(8, pmix_globals.debug_output, + "waiting for blocking recv of %"PRIsize_t" bytes", size); + + while (cnt < size) { + int retval = recv(sd, (char *)data+cnt, size-cnt, MSG_WAITALL); + + /* remote closed connection */ + if (retval == 0) { + pmix_output_verbose(8, pmix_globals.debug_output, + "ptl:base:recv_blocking: remote closed connection"); + return PMIX_ERR_UNREACH; + } + + /* handle errors */ + if (retval < 0) { + if (EAGAIN == pmix_socket_errno || + EWOULDBLOCK == pmix_socket_errno) { + /* just cycle and let it try again */ + pmix_output_verbose(8, pmix_globals.debug_output, + "blocking_recv received error %d:%s from remote - cycling", + pmix_socket_errno, strerror(pmix_socket_errno)); + continue; + } + if (pmix_socket_errno != EINTR ) { + /* If we overflow the listen backlog, it's + possible that even though we finished the three + way handshake, the remote host was unable to + transition the connection from half connected + (received the initial SYN) to fully connected + (in the listen backlog). We likely won't see + the failure until we try to receive, due to + timing and the like. The first thing we'll get + in that case is a RST packet, which receive + will turn into a connection reset by peer + errno. In that case, leave the socket in + CONNECT_ACK and propogate the error up to + recv_connect_ack, who will try to establish the + connection again */ + pmix_output_verbose(8, pmix_globals.debug_output, + "blocking_recv received error %d:%s from remote - aborting", + pmix_socket_errno, strerror(pmix_socket_errno)); + return PMIX_ERR_UNREACH; + } + continue; + } + cnt += retval; + } + + pmix_output_verbose(8, pmix_globals.debug_output, + "blocking receive complete from remote"); + return PMIX_SUCCESS; +} + +#define PMIX_MAX_RETRIES 10 + +pmix_status_t pmix_ptl_base_connect(struct sockaddr_storage *addr, + pmix_socklen_t addrlen, int *fd) +{ + int sd = -1; + int retries = 0; + + pmix_output_verbose(2, pmix_globals.debug_output, + "ptl_base_connect: attempting to connect to server"); + + while (retries < PMIX_MAX_RETRIES) { + retries++; + /* Create the new socket */ + sd = socket(addr->ss_family, SOCK_STREAM, 0); + if (sd < 0) { + pmix_output(0, "pmix:create_socket: socket() failed: %s (%d)\n", + strerror(pmix_socket_errno), + pmix_socket_errno); + continue; + } + pmix_output_verbose(2, pmix_globals.debug_output, + "usock_peer_try_connect: attempting to connect to server on socket %d", sd); + /* try to connect */ + if (connect(sd, (struct sockaddr*)addr, addrlen) < 0) { + if (pmix_socket_errno == ETIMEDOUT) { + /* The server may be too busy to accept new connections */ + pmix_output_verbose(2, pmix_globals.debug_output, + "timeout connecting to server"); + CLOSE_THE_SOCKET(sd); + continue; + } + + /* Some kernels (Linux 2.6) will automatically software + abort a connection that was ECONNREFUSED on the last + attempt, without even trying to establish the + connection. Handle that case in a semi-rational + way by trying twice before giving up */ + if (ECONNABORTED == pmix_socket_errno) { + pmix_output_verbose(2, pmix_globals.debug_output, + "connection to server aborted by OS - retrying"); + CLOSE_THE_SOCKET(sd); + continue; + } else { + pmix_output_verbose(2, pmix_globals.debug_output, + "Connect failed: %s (%d)", strerror(pmix_socket_errno), + pmix_socket_errno); + CLOSE_THE_SOCKET(sd); + continue; + } + } else { + /* otherwise, the connect succeeded - so break out of the loop */ + break; + } + } + + if (retries == PMIX_MAX_RETRIES || sd < 0){ + /* We were unsuccessful in establishing this connection, and are + * not likely to suddenly become successful */ + if (0 <= sd) { + CLOSE_THE_SOCKET(sd); + } + return PMIX_ERR_UNREACH; + } + *fd = sd; + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c new file mode 100644 index 0000000000..2b67cb4631 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c @@ -0,0 +1,220 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2009 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** @file: + * + */ +#include + +#include + +#ifdef HAVE_STRING_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif + +#include "src/mca/mca.h" +#include "src/mca/base/base.h" +#include "src/mca/base/pmix_mca_base_var.h" +#include "src/mca/base/pmix_mca_base_framework.h" +#include "src/class/pmix_list.h" +#include "src/mca/ptl/base/base.h" + +/* + * The following file was created by configure. It contains extern + * statements and the definition of an array of pointers to each + * component's public mca_base_component_t struct. + */ + +#include "src/mca/ptl/base/static-components.h" + +/* Instantiate the global vars */ +pmix_ptl_globals_t pmix_ptl_globals = {{{0}}}; +pmix_ptl_API_t pmix_ptl = { + .set_notification_cbfunc = pmix_ptl_stub_set_notification_cbfunc, + .get_available_modules = pmix_ptl_stub_get_available_modules, + .send_recv = pmix_ptl_stub_send_recv, + .send_oneway = pmix_ptl_stub_send_oneway, + .connect_to_peer = pmix_ptl_stub_connect_to_peer, + .start_listening = pmix_ptl_base_start_listening, + .stop_listening = pmix_ptl_base_stop_listening +}; + +static int pmix_ptl_register(pmix_mca_base_register_flag_t flags) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t pmix_ptl_close(void) +{ + if (!pmix_ptl_globals.initialized) { + return PMIX_SUCCESS; + } + pmix_ptl_globals.initialized = false; + + /* ensure the listen thread has been shut down */ + pmix_ptl.stop_listening(); + + /* the components will cleanup when closed */ + PMIX_DESTRUCT(&pmix_ptl_globals.actives); + PMIX_LIST_DESTRUCT(&pmix_ptl_globals.posted_recvs); + PMIX_LIST_DESTRUCT(&pmix_ptl_globals.listeners); + + return pmix_mca_base_framework_components_close(&pmix_ptl_base_framework, NULL); +} + +static pmix_status_t pmix_ptl_open(pmix_mca_base_open_flag_t flags) +{ + /* initialize globals */ + pmix_ptl_globals.initialized = true; + PMIX_CONSTRUCT(&pmix_ptl_globals.actives, pmix_list_t); + PMIX_CONSTRUCT(&pmix_ptl_globals.posted_recvs, pmix_list_t); + pmix_ptl_globals.listen_thread_active = false; + PMIX_CONSTRUCT(&pmix_ptl_globals.listeners, pmix_list_t); + + /* Open up all available components */ + return pmix_mca_base_framework_components_open(&pmix_ptl_base_framework, flags); +} + +PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, ptl, "PMIx Transfer Layer", + pmix_ptl_register, pmix_ptl_open, pmix_ptl_close, + mca_ptl_base_static_components, 0); + +/*** INSTANTIATE INTERNAL CLASSES ***/ +PMIX_CLASS_INSTANCE(pmix_ptl_base_active_t, + pmix_list_item_t, + NULL, NULL); + +static void scon(pmix_ptl_send_t *p) +{ + memset(&p->hdr, 0, sizeof(pmix_ptl_hdr_t)); + p->hdr.tag = UINT32_MAX; + p->hdr.nbytes = 0; + p->data = NULL; + p->hdr_sent = false; + p->sdptr = NULL; + p->sdbytes = 0; +} +static void sdes(pmix_ptl_send_t *p) +{ + if (NULL != p->data) { + PMIX_RELEASE(p->data); + } +} +PMIX_CLASS_INSTANCE(pmix_ptl_send_t, + pmix_list_item_t, + scon, sdes); + +static void rcon(pmix_ptl_recv_t *p) +{ + memset(&p->hdr, 0, sizeof(pmix_ptl_hdr_t)); + p->hdr.tag = UINT32_MAX; + p->hdr.nbytes = 0; + p->data = NULL; + p->hdr_recvd = false; + p->rdptr = NULL; + p->rdbytes = 0; +} +PMIX_CLASS_INSTANCE(pmix_ptl_recv_t, + pmix_list_item_t, + rcon, NULL); + +static void prcon(pmix_ptl_posted_recv_t *p) +{ + p->tag = UINT32_MAX; + p->cbfunc = NULL; + p->cbdata = NULL; +} +PMIX_CLASS_INSTANCE(pmix_ptl_posted_recv_t, + pmix_list_item_t, + prcon, NULL); + + +static void srcon(pmix_ptl_sr_t *p) +{ + p->bfr = NULL; + p->cbfunc = NULL; + p->cbdata = NULL; +} +PMIX_CLASS_INSTANCE(pmix_ptl_sr_t, + pmix_object_t, + srcon, NULL); + +static void pccon(pmix_pending_connection_t *p) +{ + memset(p->nspace, 0, PMIX_MAX_NSLEN+1); + p->info = NULL; + p->ninfo = 0; + p->bfrop = NULL; + p->psec = NULL; + p->ptl = NULL; + p->cred = NULL; +} +static void pcdes(pmix_pending_connection_t *p) +{ + if (NULL != p->info) { + PMIX_INFO_FREE(p->info, p->ninfo); + } + if (NULL != p->bfrop) { + free(p->bfrop); + } + if (NULL != p->psec) { + free(p->psec); + } + if (NULL != p->cred) { + free(p->cred); + } +} +PMIX_CLASS_INSTANCE(pmix_pending_connection_t, + pmix_object_t, + pccon, pcdes); + +static void lcon(pmix_listener_t *p) +{ + p->socket = -1; + p->varname = NULL; + p->uri = NULL; + p->owner_given = false; + p->group_given = false; + p->mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH; +} +static void ldes(pmix_listener_t *p) +{ + if (0 <= p->socket) { + CLOSE_THE_SOCKET(p->socket); + } + if (NULL != p->varname) { + free(p->varname); + } + if (NULL != p->uri) { + free(p->uri); + } +} +PMIX_CLASS_INSTANCE(pmix_listener_t, + pmix_list_item_t, + lcon, ldes); + +PMIX_CLASS_INSTANCE(pmix_ptl_queue_t, + pmix_object_t, + NULL, NULL); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_listener.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_listener.c new file mode 100644 index 0000000000..f9a5bd4be5 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_listener.c @@ -0,0 +1,281 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2016 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2015 Artem Y. Polyakov . + * All rights reserved. + * Copyright (c) 2016 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#include +#include + +#include +#include "src/include/pmix_globals.h" + +#ifdef HAVE_STRING_H +#include +#endif +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_SYS_UN_H +#include +#endif +#ifdef HAVE_SYS_UIO_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#include +#include +#include PMIX_EVENT_HEADER +#include + +#include "src/class/pmix_list.h" +#include "src/util/error.h" +#include "src/util/fd.h" +#include "src/util/output.h" +#include "src/util/pmix_environ.h" + + #include "src/mca/ptl/base/base.h" + +// local functions for connection support +static void* listen_thread(void *obj); +static pthread_t engine; +static bool setup_complete = false; + +/* Cycle across all available plugins and provide them with + * an opportunity to register rendezvous points (server-side + * function). Function is to return PMIX_SUCCESS if at least + * one rendezvous can be defined. */ +static pmix_status_t setup_listeners(pmix_info_t *info, size_t ninfo, bool *need_listener) +{ + pmix_ptl_base_active_t *active; + pmix_status_t rc; + + if (!pmix_ptl_globals.initialized) { + return PMIX_ERR_INIT; + } + + PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) { + if (NULL != active->component->setup_listener) { + rc = active->component->setup_listener(info, ninfo, need_listener); + if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) { + return rc; + } + } + } + /* we must have at least one listener */ + if (0 == pmix_list_get_size(&pmix_ptl_globals.listeners)) { + return PMIX_ERR_INIT; + } + return PMIX_SUCCESS; +} + +/* + * start listening thread + */ +pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo) +{ + pmix_status_t rc; + bool need_listener = false; + + /* setup the listeners */ + if (!setup_complete) { + if (PMIX_SUCCESS != (rc = setup_listeners(info, ninfo, &need_listener))) { + return rc; + } + } + setup_complete = true; + + /* if we don't need a listener thread, then we are done */ + if (!need_listener) { + return PMIX_SUCCESS; + } + + /*** spawn internal listener thread */ + if (0 > pipe(pmix_ptl_globals.stop_thread)) { + PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO); + return PMIX_ERR_OUT_OF_RESOURCE; + } + /* Make sure the pipe FDs are set to close-on-exec so that + they don't leak into children */ + if (pmix_fd_set_cloexec(pmix_ptl_globals.stop_thread[0]) != PMIX_SUCCESS || + pmix_fd_set_cloexec(pmix_ptl_globals.stop_thread[1]) != PMIX_SUCCESS) { + PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO); + close(pmix_ptl_globals.stop_thread[0]); + close(pmix_ptl_globals.stop_thread[1]); + return PMIX_ERR_OUT_OF_RESOURCE; + } + /* fork off the listener thread */ + pmix_ptl_globals.listen_thread_active = true; + if (0 > pthread_create(&engine, NULL, listen_thread, NULL)) { + pmix_ptl_globals.listen_thread_active = false; + return PMIX_ERROR; + } + + return PMIX_SUCCESS; +} + +void pmix_ptl_base_stop_listening(void) +{ + int i; + pmix_listener_t *lt; + + pmix_output_verbose(8, pmix_globals.debug_output, + "listen_thread: shutdown"); + + if (!pmix_ptl_globals.listen_thread_active) { + /* nothing we can do */ + return; + } + + /* mark it as inactive */ + pmix_ptl_globals.listen_thread_active = false; + /* use the block to break it loose just in + * case the thread is blocked in a call to select for + * a long time */ + i=1; + if (0 > write(pmix_ptl_globals.stop_thread[1], &i, sizeof(int))) { + return; + } + /* wait for thread to exit */ + pthread_join(engine, NULL); + /* close the sockets to remove the connection points */ + PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) { + CLOSE_THE_SOCKET(lt->socket); + lt->socket = -1; + } +} + +static void* listen_thread(void *obj) +{ + int rc, max, accepted_connections; + socklen_t addrlen = sizeof(struct sockaddr_storage); + pmix_pending_connection_t *pending_connection; + struct timeval timeout; + fd_set readfds; + pmix_listener_t *lt; + + pmix_output_verbose(8, pmix_globals.debug_output, + "listen_thread: active"); + + + while (pmix_ptl_globals.listen_thread_active) { + FD_ZERO(&readfds); + max = -1; + PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) { + FD_SET(lt->socket, &readfds); + max = (lt->socket > max) ? lt->socket : max; + } + /* add the stop_thread fd */ + FD_SET(pmix_ptl_globals.stop_thread[0], &readfds); + max = (pmix_ptl_globals.stop_thread[0] > max) ? pmix_ptl_globals.stop_thread[0] : max; + + /* set timeout interval */ + timeout.tv_sec = 2; + timeout.tv_usec = 0; + + /* Block in a select to avoid hammering the cpu. If a connection + * comes in, we'll get woken up right away. + */ + rc = select(max + 1, &readfds, NULL, NULL, &timeout); + if (!pmix_ptl_globals.listen_thread_active) { + /* we've been asked to terminate */ + close(pmix_ptl_globals.stop_thread[0]); + close(pmix_ptl_globals.stop_thread[1]); + return NULL; + } + if (rc < 0) { + continue; + } + + /* Spin accepting connections until all active listen sockets + * do not have any incoming connections, pushing each connection + * onto the event queue for processing + */ + do { + accepted_connections = 0; + PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) { + + /* according to the man pages, select replaces the given descriptor + * set with a subset consisting of those descriptors that are ready + * for the specified operation - in this case, a read. So we need to + * first check to see if this file descriptor is included in the + * returned subset + */ + if (0 == FD_ISSET(lt->socket, &readfds)) { + /* this descriptor is not included */ + continue; + } + + /* this descriptor is ready to be read, which means a connection + * request has been received - so harvest it. All we want to do + * here is accept the connection and push the info onto the event + * library for subsequent processing - we don't want to actually + * process the connection here as it takes too long, and so the + * OS might start rejecting connections due to timeout. + */ + pending_connection = PMIX_NEW(pmix_pending_connection_t); + pending_connection->protocol = lt->protocol; + pending_connection->ptl = lt->ptl; + event_assign(&pending_connection->ev, pmix_globals.evbase, -1, + EV_WRITE, lt->cbfunc, pending_connection); + pending_connection->sd = accept(lt->socket, + (struct sockaddr*)&(pending_connection->addr), + &addrlen); + if (pending_connection->sd < 0) { + PMIX_RELEASE(pending_connection); + if (pmix_socket_errno != EAGAIN || + pmix_socket_errno != EWOULDBLOCK) { + if (EMFILE == pmix_socket_errno || + ENOBUFS == pmix_socket_errno || + ENOMEM == pmix_socket_errno) { + PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); + } else if (EINVAL == pmix_socket_errno || + EINTR == pmix_socket_errno) { + /* race condition at finalize */ + goto done; + } else if (ECONNABORTED == pmix_socket_errno) { + /* they aborted the attempt */ + continue; + } else { + pmix_output(0, "listen_thread: accept() failed: %s (%d).", + strerror(pmix_socket_errno), pmix_socket_errno); + } + goto done; + } + continue; + } + + pmix_output_verbose(8, pmix_globals.debug_output, + "listen_thread: new connection: (%d, %d)", + pending_connection->sd, pmix_socket_errno); + /* activate the event */ + event_active(&pending_connection->ev, EV_WRITE, 1); + accepted_connections++; + } + } while (accepted_connections > 0); + } + + done: + pmix_ptl_globals.listen_thread_active = false; + return NULL; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_select.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_select.c new file mode 100644 index 0000000000..cee50a0325 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_select.c @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include + +#include "src/mca/mca.h" +#include "src/mca/base/base.h" + +#include "src/mca/ptl/base/base.h" + +static bool selected = false; + +/* Function for selecting a prioritized list of components + * from all those that are available. */ +int pmix_ptl_base_select(void) +{ + pmix_mca_base_component_list_item_t *cli = NULL; + pmix_ptl_base_component_t *component = NULL; + pmix_ptl_base_active_t *newactive, *active; + pmix_mca_base_module_t *mod; + int pri; + bool inserted; + + if (selected) { + /* ensure we don't do this twice */ + return PMIX_SUCCESS; + } + selected = true; + + /* Query all available components and ask if they have a module */ + PMIX_LIST_FOREACH(cli, &pmix_ptl_base_framework.framework_components, pmix_mca_base_component_list_item_t) { + component = (pmix_ptl_base_component_t *) cli->cli_component; + + pmix_output_verbose(5, pmix_ptl_base_framework.framework_output, + "mca:ptl:select: checking available component %s", + component->base.pmix_mca_component_name); + + /* get the module for this component */ + if (PMIX_SUCCESS != component->base.pmix_mca_query_component(&mod, &pri)) { + continue; + } + + /* add to our prioritized list of available actives */ + newactive = PMIX_NEW(pmix_ptl_base_active_t); + newactive->pri = component->priority; + newactive->component = component; + newactive->module = (pmix_ptl_module_t*)mod; + + /* maintain priority order */ + inserted = false; + PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) { + if (newactive->pri > active->pri) { + pmix_list_insert_pos(&pmix_ptl_globals.actives, + (pmix_list_item_t*)active, &newactive->super); + inserted = true; + break; + } + } + if (!inserted) { + /* must be lowest priority - add to end */ + pmix_list_append(&pmix_ptl_globals.actives, &newactive->super); + } + } + + if (4 < pmix_output_get_verbosity(pmix_ptl_base_framework.framework_output)) { + pmix_output(0, "Final PTL priorities"); + /* show the prioritized list */ + PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) { + pmix_output(0, "\tPTL: %s Priority: %d", + active->component->base.pmix_mca_component_name, active->pri); + } + } + + return PMIX_SUCCESS;; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/usock/usock_sendrecv.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c similarity index 79% rename from opal/mca/pmix/pmix2x/pmix/src/usock/usock_sendrecv.c rename to opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c index 8b878b0e2b..b3c5ce45ae 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/usock/usock_sendrecv.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c @@ -44,7 +44,7 @@ #include "src/server/pmix_server_ops.h" #include "src/util/error.h" -#include "usock.h" +#include "src/mca/ptl/base/base.h" static uint32_t current_tag = 1; // 0 is reserved for system purposes @@ -168,7 +168,7 @@ static pmix_status_t send_bytes(int sd, char **buf, size_t *remain) goto exit; } /* we hit an error and cannot progress this message */ - pmix_output(0, "pmix_usock_msg_send_bytes: write failed: %s (%d) [sd = %d]", + pmix_output(0, "pmix_ptl_base_msg_send_bytes: write failed: %s (%d) [sd = %d]", strerror(pmix_socket_errno), pmix_socket_errno, sd); ret = PMIX_ERR_UNREACH; @@ -216,7 +216,7 @@ static pmix_status_t read_bytes(int sd, char **buf, size_t *remain) * to abort this message */ pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_usock_msg_recv: readv failed: %s (%d)", + "pmix_ptl_base_msg_recv: readv failed: %s (%d)", strerror(pmix_socket_errno), pmix_socket_errno); ret = PMIX_ERR_UNREACH; @@ -240,25 +240,26 @@ exit: * A file descriptor is available/ready for send. Check the state * of the socket and take the appropriate action. */ -void pmix_usock_send_handler(int sd, short flags, void *cbdata) +void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata) { pmix_peer_t *peer = (pmix_peer_t*)cbdata; - pmix_usock_send_t *msg = peer->send_msg; + pmix_ptl_send_t *msg = peer->send_msg; pmix_status_t rc; pmix_output_verbose(2, pmix_globals.debug_output, - "sock:send_handler SENDING TO PEER %s:%d tag %d with %s msg", + "ptl:base:send_handler SENDING TO PEER %s:%d tag %u with %s msg", peer->info->nptr->nspace, peer->info->rank, - (NULL == msg) ? UINT_MAX : msg->hdr.tag, + (NULL == msg) ? UINT_MAX : ntohl(msg->hdr.tag), (NULL == msg) ? "NULL" : "NON-NULL"); + if (NULL != msg) { if (!msg->hdr_sent) { pmix_output_verbose(2, pmix_globals.debug_output, - "usock:send_handler SENDING HEADER"); + "ptl:base:send_handler SENDING HEADER"); if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) { /* header is completely sent */ pmix_output_verbose(2, pmix_globals.debug_output, - "usock:send_handler HEADER SENT"); + "ptl:base:send_handler HEADER SENT"); msg->hdr_sent = true; /* setup to send the data */ if (NULL == msg->data) { @@ -269,14 +270,14 @@ void pmix_usock_send_handler(int sd, short flags, void *cbdata) } else { /* send the data as a single block */ msg->sdptr = msg->data->base_ptr; - msg->sdbytes = msg->hdr.nbytes; + msg->sdbytes = ntohl(msg->hdr.nbytes); } /* fall thru and let the send progress */ } else if (PMIX_ERR_RESOURCE_BUSY == rc || PMIX_ERR_WOULD_BLOCK == rc) { /* exit this event and let the event lib progress */ pmix_output_verbose(2, pmix_globals.debug_output, - "usock:send_handler RES BUSY OR WOULD BLOCK"); + "ptl:base:send_handler RES BUSY OR WOULD BLOCK"); return; } else { // report the error @@ -291,22 +292,22 @@ void pmix_usock_send_handler(int sd, short flags, void *cbdata) if (msg->hdr_sent) { pmix_output_verbose(2, pmix_globals.debug_output, - "usock:send_handler SENDING BODY OF MSG"); + "ptl:base:send_handler SENDING BODY OF MSG"); if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) { // message is complete pmix_output_verbose(2, pmix_globals.debug_output, - "usock:send_handler BODY SENT"); + "ptl:base:send_handler BODY SENT"); PMIX_RELEASE(msg); peer->send_msg = NULL; } else if (PMIX_ERR_RESOURCE_BUSY == rc || PMIX_ERR_WOULD_BLOCK == rc) { /* exit this event and let the event lib progress */ pmix_output_verbose(2, pmix_globals.debug_output, - "usock:send_handler RES BUSY OR WOULD BLOCK"); + "ptl:base:send_handler RES BUSY OR WOULD BLOCK"); return; } else { // report the error - pmix_output(0, "pmix_usock_peer_send_handler: unable to send message ON SOCKET %d", + pmix_output(0, "ptl:base:peer_send_handler: unable to send message ON SOCKET %d", peer->sd); event_del(&peer->send_event); peer->send_ev_active = false; @@ -324,7 +325,7 @@ void pmix_usock_send_handler(int sd, short flags, void *cbdata) * wait for another send_event to fire before doing so. This gives * us a chance to service any pending recvs. */ - peer->send_msg = (pmix_usock_send_t*) + peer->send_msg = (pmix_ptl_send_t*) pmix_list_remove_first(&peer->send_queue); } @@ -340,14 +341,17 @@ void pmix_usock_send_handler(int sd, short flags, void *cbdata) * of the connection with the peer. */ -void pmix_usock_recv_handler(int sd, short flags, void *cbdata) +void pmix_ptl_base_recv_handler(int sd, short flags, void *cbdata) { pmix_status_t rc; pmix_peer_t *peer = (pmix_peer_t*)cbdata; - pmix_usock_recv_t *msg = NULL; + pmix_ptl_recv_t *msg = NULL; + pmix_ptl_hdr_t hdr; + size_t nbytes; + char *ptr; pmix_output_verbose(2, pmix_globals.debug_output, - "usock:recv:handler called with peer %s:%d", + "ptl:base:recv:handler called with peer %s:%d", (NULL == peer) ? "NULL" : peer->info->nptr->nspace, (NULL == peer) ? PMIX_RANK_UNDEF : peer->info->rank); @@ -357,26 +361,36 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata) /* allocate a new message and setup for recv */ if (NULL == peer->recv_msg) { pmix_output_verbose(2, pmix_globals.debug_output, - "usock:recv:handler allocate new recv msg"); - peer->recv_msg = PMIX_NEW(pmix_usock_recv_t); + "ptl:base:recv:handler allocate new recv msg"); + peer->recv_msg = PMIX_NEW(pmix_ptl_recv_t); if (NULL == peer->recv_msg) { - pmix_output(0, "usock_recv_handler: unable to allocate recv message\n"); + pmix_output(0, "sptl:base:recv_handler: unable to allocate recv message\n"); goto err_close; } peer->recv_msg->peer = peer; // provide a handle back to the peer object /* start by reading the header */ peer->recv_msg->rdptr = (char*)&peer->recv_msg->hdr; - peer->recv_msg->rdbytes = sizeof(pmix_usock_hdr_t); + peer->recv_msg->rdbytes = sizeof(pmix_ptl_hdr_t); } msg = peer->recv_msg; msg->sd = sd; /* if the header hasn't been completely read, read it */ if (!msg->hdr_recvd) { - pmix_output_verbose(2, pmix_globals.debug_output, - "usock:recv:handler read hdr on socket %d", peer->sd); - if (PMIX_SUCCESS == (rc = read_bytes(peer->sd, &msg->rdptr, &msg->rdbytes))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "ptl:base:recv:handler read hdr on socket %d", peer->sd); + nbytes = sizeof(pmix_ptl_hdr_t); + ptr = (char*)&hdr; + if (PMIX_SUCCESS == (rc = read_bytes(peer->sd, &ptr, &nbytes))) { /* completed reading the header */ peer->recv_msg->hdr_recvd = true; + /* convert the hdr to host format */ + peer->recv_msg->hdr.pindex = ntohl(hdr.pindex); + peer->recv_msg->hdr.tag = ntohl(hdr.tag); + peer->recv_msg->hdr.nbytes = ntohl(hdr.nbytes); + pmix_output_verbose(2, pmix_globals.debug_output, + "RECVD MSG FOR TAG %d SIZE %d", + (int)peer->recv_msg->hdr.tag, + (int)peer->recv_msg->hdr.nbytes); /* if this is a zero-byte message, then we are done */ if (0 == peer->recv_msg->hdr.nbytes) { pmix_output_verbose(2, pmix_globals.debug_output, @@ -388,7 +402,7 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata) peer->recv_msg->rdbytes = 0; } else { pmix_output_verbose(2, pmix_globals.debug_output, - "usock:recv:handler allocate data region of size %lu", + "ptl:base:recv:handler allocate data region of size %lu", (unsigned long)peer->recv_msg->hdr.nbytes); /* allocate the data region */ peer->recv_msg->data = (char*)malloc(peer->recv_msg->hdr.nbytes); @@ -407,7 +421,7 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata) * and let the caller know */ pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_usock_msg_recv: peer closed connection"); + "ptl:base:msg_recv: peer closed connection"); goto err_close; } } @@ -436,7 +450,7 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata) * and let the caller know */ pmix_output_verbose(2, pmix_globals.debug_output, - "pmix_usock_msg_recv: peer closed connection"); + "ptl:base:msg_recv: peer closed connection"); goto err_close; } } @@ -459,11 +473,44 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata) lost_connection(peer, PMIX_ERR_UNREACH); } -void pmix_usock_send_recv(int fd, short args, void *cbdata) +void pmix_ptl_base_send(int sd, short args, void *cbdata) { - pmix_usock_sr_t *ms = (pmix_usock_sr_t*)cbdata; - pmix_usock_posted_recv_t *req; - pmix_usock_send_t *snd; + pmix_ptl_queue_t *queue = (pmix_ptl_queue_t*)cbdata; + pmix_ptl_send_t *snd; + pmix_output_verbose(2, pmix_globals.debug_output, + "[%s:%d] queue callback called: reply to %s:%d on tag %d", + __FILE__, __LINE__, + (queue->peer)->info->nptr->nspace, + (queue->peer)->info->rank, (queue->tag)); + snd = PMIX_NEW(pmix_ptl_send_t); + snd->hdr.pindex = htonl(pmix_globals.pindex); + snd->hdr.tag = htonl(queue->tag); + snd->hdr.nbytes = htonl((queue->buf)->bytes_used); + snd->data = (queue->buf); + /* always start with the header */ + snd->sdptr = (char*)&snd->hdr; + snd->sdbytes = sizeof(pmix_ptl_hdr_t); + + /* if there is no message on-deck, put this one there */ + if (NULL == (queue->peer)->send_msg) { + (queue->peer)->send_msg = snd; + } else { + /* add it to the queue */ + pmix_list_append(&(queue->peer)->send_queue, &snd->super); + } + /* ensure the send event is active */ + if (!(queue->peer)->send_ev_active) { + event_add(&(queue->peer)->send_event, 0); + (queue->peer)->send_ev_active = true; + } + PMIX_RELEASE(queue); +} + +void pmix_ptl_base_send_recv(int fd, short args, void *cbdata) +{ + pmix_ptl_sr_t *ms = (pmix_ptl_sr_t*)cbdata; + pmix_ptl_posted_recv_t *req; + pmix_ptl_send_t *snd; uint32_t tag; /* set the tag */ @@ -471,7 +518,7 @@ void pmix_usock_send_recv(int fd, short args, void *cbdata) if (NULL != ms->cbfunc) { /* if a callback msg is expected, setup a recv for it */ - req = PMIX_NEW(pmix_usock_posted_recv_t); + req = PMIX_NEW(pmix_ptl_posted_recv_t); /* take the next tag in the sequence */ if (UINT32_MAX == current_tag ) { current_tag = 1; @@ -484,17 +531,20 @@ void pmix_usock_send_recv(int fd, short args, void *cbdata) /* add it to the list of recvs - we cannot have unexpected messages * in this subsystem as the server never sends us something that * we didn't previously request */ - pmix_list_prepend(&pmix_usock_globals.posted_recvs, &req->super); + pmix_list_prepend(&pmix_ptl_globals.posted_recvs, &req->super); } - snd = PMIX_NEW(pmix_usock_send_t); - snd->hdr.pindex = pmix_globals.pindex; - snd->hdr.tag = tag; - snd->hdr.nbytes = ms->bfr->bytes_used; + pmix_output_verbose(2, pmix_globals.debug_output, + "QUEIENG MSG TO SERVER OF SIZE %d", + (int)ms->bfr->bytes_used); + snd = PMIX_NEW(pmix_ptl_send_t); + snd->hdr.pindex = htonl(pmix_globals.pindex); + snd->hdr.tag = htonl(tag); + snd->hdr.nbytes = htonl(ms->bfr->bytes_used); snd->data = ms->bfr; /* always start with the header */ snd->sdptr = (char*)&snd->hdr; - snd->sdbytes = sizeof(pmix_usock_hdr_t); + snd->sdbytes = sizeof(pmix_ptl_hdr_t); /* if there is no message on-deck, put this one there */ if (NULL == ms->peer->send_msg) { @@ -512,10 +562,10 @@ void pmix_usock_send_recv(int fd, short args, void *cbdata) PMIX_RELEASE(ms); } -void pmix_usock_process_msg(int fd, short flags, void *cbdata) +void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata) { - pmix_usock_recv_t *msg = (pmix_usock_recv_t*)cbdata; - pmix_usock_posted_recv_t *rcv; + pmix_ptl_recv_t *msg = (pmix_ptl_recv_t*)cbdata; + pmix_ptl_posted_recv_t *rcv; pmix_buffer_t buf; pmix_output_verbose(5, pmix_globals.debug_output, @@ -523,7 +573,7 @@ void pmix_usock_process_msg(int fd, short flags, void *cbdata) (int)msg->hdr.nbytes, msg->hdr.tag, msg->sd); /* see if we have a waiting recv for this message */ - PMIX_LIST_FOREACH(rcv, &pmix_usock_globals.posted_recvs, pmix_usock_posted_recv_t) { + PMIX_LIST_FOREACH(rcv, &pmix_ptl_globals.posted_recvs, pmix_ptl_posted_recv_t) { pmix_output_verbose(5, pmix_globals.debug_output, "checking msg on tag %u for tag %u", msg->hdr.tag, rcv->tag); @@ -545,7 +595,7 @@ void pmix_usock_process_msg(int fd, short flags, void *cbdata) PMIX_DESTRUCT(&buf); // free's the msg data /* also done with the recv, if not a wildcard or the error tag */ if (UINT32_MAX != rcv->tag && 0 != rcv->tag) { - pmix_list_remove_item(&pmix_usock_globals.posted_recvs, &rcv->super); + pmix_list_remove_item(&pmix_ptl_globals.posted_recvs, &rcv->super); PMIX_RELEASE(rcv); } PMIX_RELEASE(msg); @@ -555,7 +605,7 @@ void pmix_usock_process_msg(int fd, short flags, void *cbdata) } /* we get here if no matching recv was found - this is an error */ - pmix_output(0, "UNEXPECTED MESSAGE tag =%d", msg->hdr.tag); + pmix_output(0, "UNEXPECTED MESSAGE tag = %d", msg->hdr.tag); PMIX_RELEASE(msg); PMIX_REPORT_EVENT(PMIX_ERROR, _notify_complete); } diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c new file mode 100644 index 0000000000..a82d4112e6 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#ifdef HAVE_UNISTD_H +#include +#endif + +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/include/pmix_globals.h" + +#include "src/mca/ptl/base/base.h" + +pmix_status_t pmix_ptl_stub_set_notification_cbfunc(pmix_ptl_cbfunc_t cbfunc) +{ + pmix_ptl_posted_recv_t *req; + + /* post a persistent recv for the special 0 tag so the client can recv + * error notifications from the server */ + req = PMIX_NEW(pmix_ptl_posted_recv_t); + if (NULL == req) { + return PMIX_ERR_NOMEM; + } + req->tag = 0; + req->cbfunc = cbfunc; + pmix_output_verbose(5, pmix_globals.debug_output, + "posting notification recv on tag %d", req->tag); + /* add it to the list of recvs - we cannot have unexpected messages + * in this subsystem as the server never sends us something that + * we didn't previously request */ + pmix_list_prepend(&pmix_ptl_globals.posted_recvs, &req->super); + return PMIX_SUCCESS; +} + +char* pmix_ptl_stub_get_available_modules(void) +{ + pmix_ptl_base_active_t *active; + char **tmp=NULL, *reply=NULL; + + if (!pmix_ptl_globals.initialized) { + return NULL; + } + + PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) { + pmix_argv_append_nosize(&tmp, active->component->base.pmix_mca_component_name); + } + if (NULL != tmp) { + reply = pmix_argv_join(tmp, ','); + pmix_argv_free(tmp); + } + return reply; +} + +pmix_status_t pmix_ptl_stub_send_recv(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_peer_t *pr = (pmix_peer_t*)peer; + + return pr->compat.ptl->send_recv(peer, bfr, cbfunc, cbdata); +} + +pmix_status_t pmix_ptl_stub_send_oneway(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_tag_t tag) +{ + pmix_peer_t *pr = (pmix_peer_t*)peer; + return pr->compat.ptl->send(peer, bfr, tag); +} + +pmix_status_t pmix_ptl_stub_connect_to_peer(struct pmix_peer_t *peer, + pmix_info_t info[], size_t ninfo) +{ + pmix_peer_t *pr = (pmix_peer_t*)peer; + pmix_ptl_base_active_t *active; + + PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) { + if (NULL != active->module->connect_to_peer) { + if (PMIX_SUCCESS == active->module->connect_to_peer(peer, info, ninfo)) { + pr->compat.ptl = active->module; + return PMIX_SUCCESS; + } + } + } + + return PMIX_ERR_UNREACH; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h new file mode 100644 index 0000000000..3681f8bb46 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h @@ -0,0 +1,199 @@ +/* -*- Mode: C; c-basic-offset:4 ; -*- */ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file + * + * Data packing subsystem. + */ + +#ifndef PMIX_PTL_H_ +#define PMIX_PTL_H_ + +#include + +#include + +#include "src/mca/mca.h" +#include "src/mca/base/pmix_mca_base_var.h" +#include "src/mca/base/pmix_mca_base_framework.h" +#include "src/buffer_ops/types.h" + +#include "ptl_types.h" + +BEGIN_C_DECLS + +/* forward declaration */ +struct pmix_peer_t; + +/* The overall objective of this framework is to provide transport + * options by which a server can communicate with a client: + * + * (a) across different versions of the library - e.g., when the + * connection handshake changes. + * + * (b) using different transports as necessitated by different + * environments. + * + * This is a mult-select framework - i.e., multiple components + * are selected and "active" at the same time. The intent is + * to have one component for each use-case, with the + * expectation that the community will do its best not to revise + * communications in manners that expand components to support (a). + * Thus, new variations should be rare, and only a few components + * will exist. + * + * The framework itself reflects the fact that any given peer + * will utilize only one messaging method. + * Thus, once a peer is identified, it will pass its version string + * to this framework's "assign_module" function, which will then + * pass it to each component until one returns a module capable of + * processing the given version. This module is then "attached" to + * the pmix_peer_t object so it can be used for all subsequent + * communication to/from that peer. + * + * Accordingly, there are two levels of APIs defined for this + * framework: + * + * (a) component level - these allow for init/finalize of the + * component, and assignment of a module to a given peer + * based on the version that peer is using + * + * (b) module level - implement send/recv/etc. Note that the + * module only needs to provide those functions that differ + * from the base functions - they don't need to duplicate + * all that code! + */ + +/**** MODULE INTERFACE DEFINITION ****/ + +/* initialize an active plugin - note that servers may have + * multiple active plugins, while clients can only have one */ +typedef pmix_status_t (*pmix_ptl_init_fn_t)(void); + +/* finalize an active plugin */ +typedef void (*pmix_ptl_finalize_fn_t)(void); + +/* (TWO-WAY) send a message to the peer, and get a response delivered + * to the specified callback function. The buffer will be free'd + * at the completion of the send, and the cbfunc will be called + * when the corresponding reply is received */ +typedef pmix_status_t (*pmix_ptl_send_recv_fn_t)(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_cbfunc_t cbfunc, + void *cbdata); + +/* (ONE-WAY) send a message to the peer. The buffer will be free'd + * at the completion of the send */ +typedef pmix_status_t (*pmix_ptl_send_fn_t)(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_tag_t tag); + +/* connect to a peer - this is a blocking function + * to establish a connection to a peer. It assigns + * the corresponding module to the peer's compat + * structure for future use */ +typedef pmix_status_t (*pmix_ptl_connect_to_peer_fn_t)(struct pmix_peer_t *peer, + pmix_info_t info[], size_t ninfo); + + +/** + * Base structure for a PTL module + */ +struct pmix_ptl_module_t { + pmix_ptl_init_fn_t init; + pmix_ptl_finalize_fn_t finalize; + pmix_ptl_send_recv_fn_t send_recv; + pmix_ptl_send_fn_t send; + pmix_ptl_connect_to_peer_fn_t connect_to_peer; +}; +typedef struct pmix_ptl_module_t pmix_ptl_module_t; + +/**** API MODULE DEFINITION ****/ +/* set the notification callback function to the provided one */ +typedef pmix_status_t (*pmix_ptl_set_notification_cbfunc_fn_t)(pmix_ptl_cbfunc_t cbfunc); + +/* get a list of available support - caller must free results + * when done. The list is returned as a comma-delimited string + * of available components in priority order, suitable for + * passing to the assign_module function */ +typedef char* (*pmix_ptl_get_available_modules_fn_t)(void); + +/* Start listening for PMIx clients (server-side function) */ +typedef pmix_status_t (*pmix_ptl_start_listening_fn_t)(pmix_info_t *info, size_t ninfo); + +/* Stop listening for PMIx clients and cleanup all rendezvous + * points (server-side function) */ +typedef void (*pmix_ptl_stop_listening_fn_t)(void); + +typedef struct { + pmix_ptl_set_notification_cbfunc_fn_t set_notification_cbfunc; + pmix_ptl_get_available_modules_fn_t get_available_modules; + pmix_ptl_send_recv_fn_t send_recv; + pmix_ptl_send_fn_t send_oneway; + pmix_ptl_connect_to_peer_fn_t connect_to_peer; + pmix_ptl_start_listening_fn_t start_listening; + pmix_ptl_stop_listening_fn_t stop_listening; +} pmix_ptl_API_t; + +PMIX_EXPORT extern pmix_ptl_API_t pmix_ptl; + + +/**** COMPONENT STRUCTURE DEFINITION ****/ + +/* define a component-level API for establishing a + * communication rendezvous point for local procs. Each active component + * would be given an opportunity to register a listener with the + * PTL base, and/or to establish their own method for handling + * connection requests. The component sets the need_listener flag + * to true if a listener thread is required - otherwise, it does _not_ + * modify this parameter */ +typedef pmix_status_t (*pmix_ptl_base_setup_listener_fn_t)(pmix_info_t info[], size_t ninfo, + bool *need_listener); + +/* + * the standard component data structure + */ +struct pmix_ptl_base_component_t { + pmix_mca_base_component_t base; + pmix_mca_base_component_data_t data; + int priority; + char* uri; + pmix_ptl_base_setup_listener_fn_t setup_listener; +}; +typedef struct pmix_ptl_base_component_t pmix_ptl_base_component_t; + + +/**** DEFINE SOME GENERAL ACCESS FUNCTIONS ****/ + + +/* + * Macro for use in components that are of type ptl + */ +#define PMIX_PTL_BASE_VERSION_1_0_0 \ + PMIX_MCA_BASE_VERSION_1_0_0("ptl", 1, 0, 0) + +END_C_DECLS + +#endif /* PMIX_PTL_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h new file mode 100644 index 0000000000..b397030c31 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h @@ -0,0 +1,257 @@ +/* -*- C -*- + * + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. + * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file + * + * Buffer management types. + */ + +#ifndef PMIX_PTL_TYPES_H_ +#define PMIX_PTL_TYPES_H_ + +#include +#include "src/include/types.h" + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_SYS_UN_H +#include +#endif +#ifdef HAVE_SYS_UIO_H +#include +#endif +#ifdef HAVE_NET_UIO_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#include PMIX_EVENT_HEADER + +#include "src/class/pmix_list.h" +#include "src/util/output.h" +#include "src/buffer_ops/types.h" + +BEGIN_C_DECLS + +// forward declaration +struct pmix_peer_t; +struct pmix_ptl_module_t; + +/**** MESSAGING STRUCTURES ****/ +typedef uint32_t pmix_ptl_tag_t; + +/* header for messages */ +typedef struct { + int32_t pindex; + pmix_ptl_tag_t tag; + size_t nbytes; +} pmix_ptl_hdr_t; + +/* define the messaging cbfunc */ +typedef void (*pmix_ptl_cbfunc_t)(struct pmix_peer_t *peer, + pmix_ptl_hdr_t *hdr, + pmix_buffer_t *buf, void *cbdata); + +/* define a callback function for notifying that server connection + * has completed */ +typedef void (*pmix_ptl_connect_cbfunc_t)(pmix_status_t status, void *cbdata); + +/* define a callback function for processing pending connections */ +typedef void (*pmix_ptl_pending_cbfunc_t)(int sd, short args, void *cbdata); + + +/* structure for sending a message */ +typedef struct { + pmix_list_item_t super; + pmix_event_t ev; + pmix_ptl_hdr_t hdr; + pmix_buffer_t *data; + bool hdr_sent; + char *sdptr; + size_t sdbytes; +} pmix_ptl_send_t; +PMIX_CLASS_DECLARATION(pmix_ptl_send_t); + +/* structure for recving a message */ +typedef struct { + pmix_list_item_t super; + pmix_event_t ev; + struct pmix_peer_t *peer; + int sd; + pmix_ptl_hdr_t hdr; + char *data; + bool hdr_recvd; + char *rdptr; + size_t rdbytes; +} pmix_ptl_recv_t; +PMIX_CLASS_DECLARATION(pmix_ptl_recv_t); + +/* structure for tracking posted recvs */ +typedef struct { + pmix_list_item_t super; + pmix_event_t ev; + uint32_t tag; + pmix_ptl_cbfunc_t cbfunc; + void *cbdata; +} pmix_ptl_posted_recv_t; +PMIX_CLASS_DECLARATION(pmix_ptl_posted_recv_t); + +/* struct for posting send/recv request */ +typedef struct { + pmix_object_t super; + volatile bool active; + pmix_event_t ev; + struct pmix_peer_t *peer; + pmix_status_t status; + pmix_buffer_t *bfr; + pmix_ptl_cbfunc_t cbfunc; + void *cbdata; +} pmix_ptl_sr_t; +PMIX_CLASS_DECLARATION(pmix_ptl_sr_t); + +typedef struct { + pmix_object_t super; + pmix_event_t ev; + struct pmix_peer_t *peer; + pmix_buffer_t *buf; + pmix_ptl_tag_t tag; +} pmix_ptl_queue_t; +PMIX_CLASS_DECLARATION(pmix_ptl_queue_t); + +/* define listener protocol types */ +typedef uint16_t pmix_listener_protocol_t; +#define PMIX_PROTOCOL_V1 0 // legacy usock +#define PMIX_PROTOCOL_V2 1 // tcp +#define PMIX_PROTOCOL_V3 2 // updated usock + +/* connection support */ +typedef struct { + pmix_object_t super; + pmix_event_t ev; + pmix_listener_protocol_t protocol; + int sd; + char nspace[PMIX_MAX_NSLEN+1]; + pmix_info_t *info; + size_t ninfo; + pmix_status_t status; + struct sockaddr_storage addr; + char *bfrop; + char *psec; + struct pmix_ptl_module_t *ptl; + pmix_bfrop_buffer_type_t buffer_type; + char *cred; + size_t len; + uid_t uid; + gid_t gid; +} pmix_pending_connection_t; +PMIX_CLASS_DECLARATION(pmix_pending_connection_t); + +/* listener objects */ +typedef struct pmix_listener_t { + pmix_list_item_t super; + pmix_listener_protocol_t protocol; + struct pmix_ptl_module_t *ptl; + int socket; + char *varname; + char *uri; + uint32_t owner; + bool owner_given; + uint32_t group; + bool group_given; + uint32_t mode; + pmix_ptl_pending_cbfunc_t cbfunc; +} pmix_listener_t; +PMIX_CLASS_DECLARATION(pmix_listener_t); + + +#define PMIX_ACTIVATE_POST_MSG(ms) \ + do { \ + pmix_output_verbose(5, pmix_globals.debug_output, \ + "[%s:%d] post msg", \ + __FILE__, __LINE__); \ + event_assign(&((ms)->ev), pmix_globals.evbase, -1, \ + EV_WRITE, pmix_ptl_base_process_msg, (ms)); \ + event_active(&((ms)->ev), EV_WRITE, 1); \ + } while (0) + +#define PMIX_SND_CADDY(c, h, s) \ + do { \ + (c) = PMIX_NEW(pmix_server_caddy_t); \ + (void)memcpy(&(c)->hdr, &(h), sizeof(pmix_ptl_hdr_t)); \ + PMIX_RETAIN((s)); \ + (c)->snd = (s); \ + } while (0) + +/* queue a message to be sent to one of our procs - must + * provide the following params: + * p - pmix_peer_t of target recipient + * t - tag to be sent to + * b - buffer to be sent + */ +#define PMIX_SERVER_QUEUE_REPLY(p, t, b) \ + do { \ + pmix_ptl_send_t *snd; \ + pmix_output_verbose(5, pmix_globals.debug_output, \ + "[%s:%d] queue callback called: reply to %s:%d on tag %d size %d", \ + __FILE__, __LINE__, \ + (p)->info->nptr->nspace, \ + (p)->info->rank, (t), (int)(b)->bytes_used); \ + snd = PMIX_NEW(pmix_ptl_send_t); \ + snd->hdr.pindex = htonl(pmix_globals.pindex); \ + snd->hdr.tag = htonl(t); \ + snd->hdr.nbytes = htonl((b)->bytes_used); \ + snd->data = (b); \ + /* always start with the header */ \ + snd->sdptr = (char*)&snd->hdr; \ + snd->sdbytes = sizeof(pmix_ptl_hdr_t); \ + /* if there is no message on-deck, put this one there */ \ + if (NULL == (p)->send_msg) { \ + (p)->send_msg = snd; \ + } else { \ + /* add it to the queue */ \ + pmix_list_append(&(p)->send_queue, &snd->super); \ + } \ + /* ensure the send event is active */ \ + if (!(p)->send_ev_active) { \ + event_add(&(p)->send_event, 0); \ + (p)->send_ev_active = true; \ + } \ + } while (0) + +#define CLOSE_THE_SOCKET(socket) \ + do { \ + if (0 <= socket) { \ + shutdown(socket, 2); \ + close(socket); \ + socket = -1; \ + } \ + } while (0) + + +END_C_DECLS + +#endif /* PMIX_PTL_TYPES_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am new file mode 100644 index 0000000000..6788aba19c --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am @@ -0,0 +1,50 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers = ptl_tcp.h +sources = \ + ptl_tcp_component.c \ + ptl_tcp.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_ptl_tcp_DSO +lib = +lib_sources = +component = mca_ptl_tcp.la +component_sources = $(headers) $(sources) +else +lib = libmca_ptl_tcp.la +lib_sources = $(headers) $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component) +mca_ptl_tcp_la_SOURCES = $(component_sources) +mca_ptl_tcp_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(lib) +libmca_ptl_tcp_la_SOURCES = $(lib_sources) +libmca_ptl_tcp_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c new file mode 100644 index 0000000000..3deb3686a2 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c @@ -0,0 +1,732 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include + +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_SYS_UIO_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#include "src/include/pmix_globals.h" +#include "src/include/pmix_socket_errno.h" +#include "src/client/pmix_client_ops.h" +#include "src/server/pmix_server_ops.h" +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/os_path.h" + +#include "src/mca/ptl/base/base.h" +#include "ptl_tcp.h" + +static pmix_status_t init(void); +static void finalize(void); +static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, + pmix_info_t *info, size_t ninfo); +static pmix_status_t send_recv(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_cbfunc_t cbfunc, + void *cbdata); +static pmix_status_t send_oneway(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_tag_t tag); + +pmix_ptl_module_t pmix_ptl_tcp_module = { + .init = init, + .finalize = finalize, + .send_recv = send_recv, + .send = send_oneway, + .connect_to_peer = connect_to_peer +}; + +static pmix_status_t recv_connect_ack(int sd); +static pmix_status_t send_connect_ack(int sd); + + +static pmix_status_t init(void) +{ + return PMIX_SUCCESS; +} + +static void finalize(void) +{ +} + +static char *pmix_getline(FILE *fp) +{ + char *ret, *buff; + char input[1024]; + + ret = fgets(input, 1024, fp); + if (NULL != ret) { + input[strlen(input)-1] = '\0'; /* remove newline */ + buff = strdup(input); + return buff; + } + + return NULL; +} + +static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, + pmix_info_t *info, size_t ninfo) +{ + char *evar, **uri; + char *filename, *host; + FILE *fp; + char *srvr, *p, *p2; + struct sockaddr_in *in; + struct sockaddr_in6 *in6; + pmix_socklen_t len; + int sd, rc; + + pmix_output_verbose(2, pmix_globals.debug_output, + "ptl:tcp: connecting to server"); + + /* see if the connection info is in the info array - if + * so, then that overrides all other options */ + + + /* if I am a client, then we need to look for the appropriate + * connection info in the environment */ + if (PMIX_PROC_CLIENT == pmix_globals.proc_type) { + if (NULL == (evar = getenv("PMIX_SERVER_URI2"))) { + /* not us */ + return PMIX_ERR_NOT_SUPPORTED; + } + + /* the URI consists of elements: + * - server nspace.rank + * - ptl rendezvous URI + */ + uri = pmix_argv_split(evar, ';'); + if (2 != pmix_argv_count(uri)) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + pmix_argv_free(uri); + return PMIX_ERR_NOT_SUPPORTED; + } + + /* set the server nspace */ + p = uri[0]; + p2 = strchr(p, '.'); + *p2 = '\0'; + ++p2; + pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t); + pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t); + (void)strncpy(pmix_client_globals.myserver.info->nptr->nspace, p, PMIX_MAX_NSLEN); + + /* set the server rank */ + pmix_client_globals.myserver.info->rank = strtoull(p2, NULL, 10); + + /* save the URI, but do not overwrite what we may have received from + * the info-key directives */ + if (NULL == mca_ptl_tcp_component.super.uri) { + mca_ptl_tcp_component.super.uri = strdup(uri[1]); + } + pmix_argv_free(uri); + + } else if (PMIX_PROC_TOOL == pmix_globals.proc_type) { + /* if we already have a URI, then look no further */ + if (NULL == mca_ptl_tcp_component.super.uri) { + /* we have to discover the connection info, + * if possible. Start by looking for the connection + * info in the expected place - if the server supports + * tool connections via TCP, then there will be a + * "contact.txt" file under the system tmpdir */ + filename = pmix_os_path(false, mca_ptl_tcp_component.tmpdir, "pmix-contact.txt", NULL); + if (NULL == filename) { + return PMIX_ERR_NOMEM; + } + fp = fopen(filename, "r"); + if (NULL == fp) { + /* if we cannot open the file, then the server must not + * be configured to support tool connections - so abort */ + free(filename); + return PMIX_ERR_UNREACH; + } + free(filename); + /* get the URI */ + srvr = pmix_getline(fp); + if (NULL == srvr) { + PMIX_ERROR_LOG(PMIX_ERR_FILE_READ_FAILURE); + fclose(fp); + return PMIX_ERR_UNREACH; + } + fclose(fp); + /* up to the first ';' is the server nspace/rank */ + if (NULL == (p = strchr(srvr, ';'))) { + /* malformed */ + free(srvr); + return PMIX_ERR_UNREACH; + } + *p = '\0'; + ++p; // move past the semicolon + /* the nspace is the section up to the '.' */ + if (NULL == (p2 = strchr(srvr, '.'))) { + /* malformed */ + free(srvr); + return PMIX_ERR_UNREACH; + } + *p2 = '\0'; + ++p2; + /* set the server nspace */ + pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t); + pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t); + (void)strncpy(pmix_client_globals.myserver.info->nptr->nspace, srvr, PMIX_MAX_NSLEN); + pmix_client_globals.myserver.info->rank = strtoull(p2, NULL, 10); + /* now parse the uti itself */ + mca_ptl_tcp_component.super.uri = strdup(p); + free(srvr); + } + } + + /* mark that we are the active module for this server */ + pmix_client_globals.myserver.compat.ptl = &pmix_ptl_tcp_module; + + /* setup the path to the daemon rendezvous point */ + memset(&mca_ptl_tcp_component.connection, 0, sizeof(struct sockaddr_storage)); + if (0 == strncmp(mca_ptl_tcp_component.super.uri, "tcp4", 4)) { + /* separate the IP address from the port */ + p = strdup(mca_ptl_tcp_component.super.uri); + p2 = strchr(&p[7], ':'); + *p2 = '\0'; + ++p2; + host = strdup(&p[7]); + /* load the address */ + in = (struct sockaddr_in*)&mca_ptl_tcp_component.connection; + in->sin_family = AF_INET; + in->sin_addr.s_addr = inet_addr(host); + if (in->sin_addr.s_addr == INADDR_NONE) { + return PMIX_ERR_BAD_PARAM; + } + in->sin_port = htons(atoi(p2)); + len = sizeof(struct sockaddr_in); + } else { + /* separate the IP address from the port */ + p = strdup(mca_ptl_tcp_component.super.uri); + p2 = strchr(&p[7], ':'); + *p2 = '\0'; + if (']' == p[strlen(p)-1]) { + p[strlen(p)-1] = '\0'; + } + if ('[' == p[7]) { + host = strdup(&p[8]); + } else { + host = strdup(&p[7]); + } + /* load the address */ + in6 = (struct sockaddr_in6*)&mca_ptl_tcp_component.connection; + in6->sin6_family = AF_INET6; + if (0 == inet_pton(AF_INET6, host, (void*)&in6->sin6_addr)) { + pmix_output (0, "ptl_tcp_parse_uri: Could not convert %s\n", host); + return PMIX_ERR_BAD_PARAM; + } + in6->sin6_port = htons(atoi(p2)); + len = sizeof(struct sockaddr_in6); + } + + /* establish the connection */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&mca_ptl_tcp_component.connection, len, &sd))) { + PMIX_ERROR_LOG(rc); + return rc; + } + pmix_client_globals.myserver.sd = sd; + + /* send our identity and any authentication credentials to the server */ + if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(sd); + return rc; + } + + /* do whatever handshake is required */ + if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(sd); + return rc; + } + + pmix_output_verbose(2, pmix_globals.debug_output, + "sock_peer_try_connect: Connection across to server succeeded"); + + /* mark the connection as made */ + pmix_globals.connected = true; + + pmix_ptl_base_set_nonblocking(sd); + + /* setup recv event */ + event_assign(&pmix_client_globals.myserver.recv_event, + pmix_globals.evbase, + pmix_client_globals.myserver.sd, + EV_READ | EV_PERSIST, + pmix_ptl_base_recv_handler, &pmix_client_globals.myserver); + event_add(&pmix_client_globals.myserver.recv_event, 0); + pmix_client_globals.myserver.recv_ev_active = true; + + /* setup send event */ + event_assign(&pmix_client_globals.myserver.send_event, + pmix_globals.evbase, + pmix_client_globals.myserver.sd, + EV_WRITE|EV_PERSIST, + pmix_ptl_base_send_handler, &pmix_client_globals.myserver); + pmix_client_globals.myserver.send_ev_active = false; + + return PMIX_SUCCESS; +} + +static pmix_status_t send_recv(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_ptl_sr_t *ms; + pmix_output_verbose(5, pmix_globals.debug_output, + "[%s:%d] post send to server", + __FILE__, __LINE__); + ms = PMIX_NEW(pmix_ptl_sr_t); + ms->peer = peer; + ms->bfr = bfr; + ms->cbfunc = cbfunc; + ms->cbdata = cbdata; + event_assign(&ms->ev, pmix_globals.evbase, -1, + EV_WRITE, pmix_ptl_base_send_recv, ms); + event_active(&ms->ev, EV_WRITE, 1); + return PMIX_SUCCESS; +} + +static pmix_status_t send_oneway(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_tag_t tag) +{ + pmix_ptl_queue_t *q; + + /* we have to transfer this to an event for thread + * safety as we need to post this message on the + * peer's send queue */ + q = PMIX_NEW(pmix_ptl_queue_t); + q->peer = peer; + q->buf = bfr; + q->tag = tag; + event_assign(&q->ev, pmix_globals.evbase, -1, + EV_WRITE, pmix_ptl_base_send, q); + event_active(&q->ev, EV_WRITE, 1); + + return PMIX_SUCCESS; +} + + +/**** SUPPORTING FUNCTIONS ****/ +static pmix_status_t send_connect_ack(int sd) +{ + char *msg; + pmix_ptl_hdr_t hdr; + size_t sdsize=0, csize=0, len; + char *cred = NULL; + char *sec; + pmix_status_t rc; + uint8_t flag; + uid_t euid; + gid_t egid; + uint32_t u32; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix:tcp SEND CONNECT ACK"); + + /* if we are a server, then we shouldn't be here */ + if (PMIX_PROC_IS_SERVER) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* setup the header */ + memset(&hdr, 0, sizeof(pmix_ptl_hdr_t)); + hdr.pindex = -1; + hdr.tag = UINT32_MAX; + + /* a security module was assigned to us during rte_init based + * on a list of available security modules provided by our + * local PMIx server, if known. Now use that module to + * get a credential, if the security system provides one. Not + * every psec module will do so, thus we must first check */ + if (PMIX_SUCCESS != (rc = pmix_psec.create_cred(&pmix_client_globals.myserver, + PMIX_PROTOCOL_V2, &cred, &len))) { + return rc; + } + + /* allow space for a marker indicating client vs tool */ + sdsize = 1; + + if (PMIX_PROC_IS_CLIENT) { + flag = 0; + /* reserve space for our nspace and rank info */ + sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(int); + } else { + flag = 1; + /* add space for our uid/gid for ACL purposes */ + sdsize += 2*sizeof(uint32_t); + } + + /* add our active sec module info */ + sec = pmix_psec.get_available_modules(); + + /* set the number of bytes to be read beyond the header */ + hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 + sizeof(uint32_t) + len; // must NULL terminate the VERSION string! + + /* create a space for our message */ + sdsize = (sizeof(hdr) + hdr.nbytes); + if (NULL == (msg = (char*)malloc(sdsize))) { + if (NULL != cred) { + free(cred); + } + free(sec); + return PMIX_ERR_OUT_OF_RESOURCE; + } + memset(msg, 0, sdsize); + + /* load the message */ + csize=0; + memcpy(msg, &hdr, sizeof(pmix_ptl_hdr_t)); + csize += sizeof(pmix_ptl_hdr_t); + + /* provide our active psec module */ + memcpy(msg+csize, sec, strlen(sec)); + csize += strlen(sec)+1; + free(sec); + + /* load the length of the credential - we put this in uint32_t + * format as that is a fixed size, and convert to network + * byte order for heterogeneity */ + u32 = htonl((uint32_t)len); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + /* load the credential */ + if (0 < u32) { + memcpy(msg+csize, cred, len); + csize += len; + } + + /* load our process type - this is a single byte, + * so no worry about heterogeneity here */ + memcpy(msg+csize, &flag, 1); + csize += 1; + + if (PMIX_PROC_IS_CLIENT) { + /* if we are a client, provide our nspace/rank */ + memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); + csize += strlen(pmix_globals.myid.nspace)+1; + /* again, need to convert */ + u32 = htonl((uint32_t)pmix_globals.myid.rank); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + } else { + /* if we are a tool, provide our uid/gid for ACL support - note + * that we have to convert so we can handle heterogeneity */ + euid = geteuid(); + u32 = htonl(euid); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + egid = getegid(); + u32 = htonl(egid); + memcpy(msg+csize, &u32, sizeof(uint32_t)); + csize += sizeof(uint32_t); + } + + /* provide our version */ + memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION)); + csize += strlen(PMIX_VERSION)+1; + + /* send the entire message across */ + if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) { + free(msg); + if (NULL != cred) { + free(cred); + } + return PMIX_ERR_UNREACH; + } + free(msg); + if (NULL != cred) { + free(cred); + } + return PMIX_SUCCESS; +} + +/* we receive a connection acknowledgement from the server, + * consisting of nothing more than a status report. If success, + * then we initiate authentication method */ +static pmix_status_t recv_connect_ack(int sd) +{ + pmix_status_t reply; + pmix_status_t rc; + struct timeval tv, save; + pmix_socklen_t sz; + bool sockopt = true; + uint32_t u32; + pmix_nspace_t *nsptr; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: RECV CONNECT ACK FROM SERVER"); + + /* get the current timeout value so we can reset to it */ + sz = sizeof(save); + if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) { + if (ENOPROTOOPT == errno) { + sockopt = false; + } else { + return PMIX_ERR_UNREACH; + } + } else { + /* set a timeout on the blocking recv so we don't hang */ + tv.tv_sec = 2; + tv.tv_usec = 0; + if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); + return PMIX_ERR_UNREACH; + } + } + + /* receive the status reply */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t)); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + reply = ntohl(u32); + /* if the status indicates an error, then we are done */ + if (PMIX_SUCCESS != reply) { + PMIX_ERROR_LOG(reply); + return reply; + } + + if (PMIX_PROC_IS_CLIENT) { + /* see if they want us to do the handshake */ + if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { + if (PMIX_SUCCESS != (rc = pmix_psec.client_handshake(&pmix_client_globals.myserver, sd))) { + return rc; + } + } else if (PMIX_SUCCESS != reply) { + return reply; + } + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: RECV CONNECT CONFIRMATION"); + + /* receive our index into the server's client array */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t)); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + pmix_globals.pindex = ntohl(u32); + } else { + /* recv our nspace */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + + /* setup required bookkeeping */ + nsptr = PMIX_NEW(pmix_nspace_t); + (void)strncpy(nsptr->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); + pmix_list_append(&pmix_globals.nspaces, &nsptr->super); + /* our rank is always zero */ + pmix_globals.myid.rank = 0; + + /* get the server's nspace and rank so we can send to it */ + pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t); + pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t); + pmix_ptl_base_recv_blocking(sd, (char*)pmix_client_globals.myserver.info->nptr->nspace, PMIX_MAX_NSLEN+1); + pmix_ptl_base_recv_blocking(sd, (char*)&(pmix_client_globals.myserver.info->rank), sizeof(int)); + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: RECV CONNECT CONFIRMATION FOR TOOL %s:%d FROM SERVER %s:%d", + pmix_globals.myid.nspace, pmix_globals.myid.rank, + pmix_client_globals.myserver.info->nptr->nspace, + pmix_client_globals.myserver.info->rank); + + /* get the returned status from the security handshake */ + pmix_ptl_base_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t)); + if (PMIX_SUCCESS != reply) { + /* see if they want us to do the handshake */ + if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { + if (NULL == pmix_psec.client_handshake) { + return PMIX_ERR_HANDSHAKE_FAILED; + } + if (PMIX_SUCCESS != (reply = pmix_psec.client_handshake(&pmix_client_globals.myserver, sd))) { + return reply; + } + /* if the handshake succeeded, then fall thru to the next step */ + } else { + return reply; + } + } + } + + if (sockopt) { + if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) { + return PMIX_ERR_UNREACH; + } + } + + return PMIX_SUCCESS; +} + +#if 0 + +DIR *cur_dirp = NULL; +struct dirent * dir_entry; +bool connect_to_system_server = false; +bool connect_to_system_first = false; +bool connection_defined = false; + +/* scan incoming info for directives */ +if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (strcmp(info[n].key, PMIX_SERVER_PIDINFO) == 0) { + server_pid = info[n].value.data.pid; + server_pid_given = true; + } else if (strcmp(info[n].key, PMIX_CONNECT_TO_SYSTEM) == 0) { + connect_to_system_server = info[n].value.data.flag; + connection_defined = true; + } else if (strcmp(info[n].key, PMIX_CONNECT_SYSTEM_FIRST) == 0) { + connect_to_system_first = info[n].value.data.flag; + connection_defined = true; + } else if (strcmp(info[n].key, PMIX_SERVER_TMPDIR) == 0 && + NULL == mytmpdir) { + mytmpdir = strdup(info[n].value.data.string); + } else if (strcmp(info[n].key, PMIX_SYSTEM_TMPDIR) == 0 && + NULL == systmpdir) { + systmpdir = strdup(info[n].value.data.string); + } + } +} + + + +/* if we are to connect solely to the system-level daemon, + * or to preferentially connect to the system-level daemon, + * or nothing was specified at all, then look to see if a + * rendezvous point in that location exists */ +if (connect_to_system_server || connect_to_system_first || !connection_defined) { + /* find the temp dir */ + if (NULL != systmpdir) { + tdir = systmpdir; + } else if (NULL == (tdir = getenv("TMPDIR"))) { + if (NULL == (tdir = getenv("TEMP"))) { + if (NULL == (tdir = getenv("TMP"))) { + tdir = "/tmp"; + } + } + } + snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/pmix.sys.%s", tdir, hostname); + /* see if the rendezvous file exists */ + if (0 != access(address.sun_path, R_OK)) { + /* if it was a requirement, then error out */ + if (connect_to_system_server) { + return PMIX_ERR_UNREACH; + } + /* otherwise, this isn't a fatal error - reset the addr */ + memset(&address, 0, sizeof(struct sockaddr_un)); + address.sun_family = AF_UNIX; + connection_defined = false; + } else { + /* connect to this server */ + connection_defined = true; + } +} + +if (!connection_defined) { + /* if we get here, then either we are to connect to + * a non-system daemon, or a system-level daemon was + * not found - so now look for the session daemon */ + + + /* find the temp dir */ + if (NULL != mytmpdir) { + tdir = mytmpdir; + } else if (NULL == (tdir = getenv("TMPDIR"))) { + if (NULL == (tdir = getenv("TEMP"))) { + if (NULL == (tdir = getenv("TMP"))) { + tdir = "/tmp"; + } + } + } + + /* if they gave us a specific pid, then look for that + * particular server - otherwise, see if there is only + * one on this node and default to it */ + if (server_pid_given) { + snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/pmix.%s.%d", tdir, hostname, server_pid); + /* if the rendezvous file doesn't exist, that's an error */ + if (0 != access(address.sun_path, R_OK)) { + return PMIX_ERR_NOT_FOUND; + } + } else { + /* open up the temp directory */ + if (NULL == (cur_dirp = opendir(tdir))) { + return PMIX_ERR_NOT_FOUND; + } + /* search the entries for something that starts with pmix.hostname */ + if (0 > asprintf(&tmp, "pmix.%s", hostname)) { + closedir(cur_dirp); + return PMIX_ERR_NOMEM; + } + evar = NULL; + while (NULL != (dir_entry = readdir(cur_dirp))) { + if (0 == strncmp(dir_entry->d_name, tmp, strlen(tmp))) { + /* found one - if more than one, then that's an error */ + if (NULL != evar) { + closedir(cur_dirp); + free(evar); + free(tmp); + return PMIX_ERR_INIT; + } + evar = strdup(dir_entry->d_name); + } + } + free(tmp); + closedir(cur_dirp); + if (NULL == evar) { + /* none found */ + return PMIX_ERR_INIT; + } + /* use the found one as our contact point */ + snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/%s", tdir, evar); + free(evar); + } +} + +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h new file mode 100644 index 0000000000..fa82625836 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_PTL_TCP_H +#define PMIX_PTL_TCP_H + +#ifdef HAVE_NETINET_IN_H +#include +#endif +#ifdef HAVE_ARPA_INET_H +#include +#endif +#ifdef HAVE_NETDB_H +#include +#endif + +#include "src/mca/ptl/ptl.h" + +BEGIN_C_DECLS + +typedef struct { + pmix_ptl_base_component_t super; + char *tmpdir; + char *if_include; + char *if_exclude; + int ipv4_port; + int ipv6_port; + bool disable_ipv4_family; + bool disable_ipv6_family; + struct sockaddr_storage connection; + char *filename; +} pmix_ptl_tcp_component_t; + +extern pmix_ptl_tcp_component_t mca_ptl_tcp_component; + +extern pmix_ptl_module_t pmix_ptl_tcp_module; + +END_C_DECLS + +#endif /* PMIX_PTL_TCP_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c new file mode 100644 index 0000000000..f8eb8ab459 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c @@ -0,0 +1,1192 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennptlee and The University + * of Tennptlee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#include +#ifdef HAVE_NETINET_IN_H +#include +#endif +#ifdef HAVE_ARPA_INET_H +#include +#endif +#ifdef HAVE_NETDB_H +#include +#endif +#include + +#include "src/include/pmix_socket_errno.h" +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/fd.h" +#include "src/util/net.h" +#include "src/util/os_path.h" +#include "src/util/parse_options.h" +#include "src/util/pif.h" +#include "src/util/show_help.h" +#include "src/util/strnlen.h" +#include "src/server/pmix_server_ops.h" + +#include "src/mca/ptl/base/base.h" +#include "src/mca/ptl/tcp/ptl_tcp.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static int component_register(void); +static int component_query(pmix_mca_base_module_t **module, int *priority); +static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, + bool *need_listener); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ + pmix_ptl_tcp_component_t mca_ptl_tcp_component = { + .super = { + .base = { + PMIX_PTL_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "tcp", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_register_component_params = component_register, + .pmix_mca_query_component = component_query + }, + .priority = 10, + .uri = NULL, + .setup_listener = setup_listener + }, + .tmpdir = NULL, + .if_include = NULL, + .if_exclude = NULL, + .ipv4_port = 0, + .ipv6_port = 0, + .disable_ipv4_family = false, + .disable_ipv6_family = true, + .filename = NULL +}; + +static char **split_and_resolve(char **orig_str, char *name); +static void connection_handler(int sd, short args, void *cbdata); +static void cnct_cbfunc(pmix_status_t status, + pmix_proc_t *proc, void *cbdata); + +static int component_register(void) +{ + pmix_mca_base_component_t *component = &mca_ptl_tcp_component.super.base; + + (void)pmix_mca_base_component_var_register(component, "server_uri", + "URI of a server a tool wishes to connect to", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_LOCAL, + &mca_ptl_tcp_component.super.uri); + + (void)pmix_mca_base_component_var_register(component, "if_include", + "Comma-delimited list of devices and/or CIDR notation of TCP networks (e.g., \"eth0,192.168.0.0/16\"). Mutually exclusive with ptl_tcp_if_exclude.", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_LOCAL, + &mca_ptl_tcp_component.if_include); + + (void)pmix_mca_base_component_var_register(component, "if_exclude", + "Comma-delimited list of devices and/or CIDR notation of TCP networks to NOT use -- all devices not matching these specifications will be used (e.g., \"eth0,192.168.0.0/16\"). If set to a non-default value, it is mutually exclusive with ptl_tcp_if_include.", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_LOCAL, + &mca_ptl_tcp_component.if_exclude); + + /* if_include and if_exclude need to be mutually exclusive */ + if (NULL != mca_ptl_tcp_component.if_include && + NULL != mca_ptl_tcp_component.if_exclude) { + /* Return ERR_NOT_AVAILABLE so that a warning message about + "open" failing is not printed */ + pmix_show_help("help-ptl-tcp.txt", "include-exclude", true, + mca_ptl_tcp_component.if_include, + mca_ptl_tcp_component.if_exclude); + return PMIX_ERR_NOT_AVAILABLE; + } + + (void)pmix_mca_base_component_var_register(component, "ipv4_port", + "IPv4 port to be used", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_4, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_ptl_tcp_component.ipv4_port); + + (void)pmix_mca_base_component_var_register(component, "ipv6_port", + "IPv6 port to be used", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_4, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_ptl_tcp_component.ipv6_port); + + (void)pmix_mca_base_component_var_register(component, "disable_ipv4_family", + "Disable the IPv4 interfaces", + PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + PMIX_INFO_LVL_4, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_ptl_tcp_component.disable_ipv4_family); + + (void)pmix_mca_base_component_var_register(component, "disable_ipv6_family", + "Disable the IPv6 interfaces", + PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + PMIX_INFO_LVL_4, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_ptl_tcp_component.disable_ipv6_family); + + return PMIX_SUCCESS; +} + +static pmix_status_t component_open(void) +{ + char *tdir; + + memset(&mca_ptl_tcp_component.connection, 0, sizeof(mca_ptl_tcp_component.connection)); + + /* check for environ-based directives + * on system tmpdir to use */ + if (NULL == (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) { + if (NULL == (tdir = getenv("TMPDIR"))) { + if (NULL == (tdir = getenv("TEMP"))) { + if (NULL == (tdir = getenv("TMP"))) { + tdir = "/tmp"; + } + } + } + } + if (NULL != tdir) { + mca_ptl_tcp_component.tmpdir = strdup(tdir); + } + return PMIX_SUCCESS; +} + + +pmix_status_t component_close(void) +{ + if (NULL != mca_ptl_tcp_component.tmpdir) { + free(mca_ptl_tcp_component.tmpdir); + } + if (NULL != mca_ptl_tcp_component.super.uri) { + free(mca_ptl_tcp_component.super.uri); + } + if (NULL != mca_ptl_tcp_component.filename) { + /* remove the file */ + unlink(mca_ptl_tcp_component.filename); + free(mca_ptl_tcp_component.filename); + } + return PMIX_SUCCESS; +} + +static int component_query(pmix_mca_base_module_t **module, int *priority) +{ + *module = (pmix_mca_base_module_t*)&pmix_ptl_tcp_module; + return PMIX_SUCCESS; +} + +/* if we are the server, then we need to discover the available + * interfaces, filter them thru any given directives, and select + * the one we will listen on for connection requests. This will + * be a loopback device by default, unless we are asked to support + * tool connections - in that case, we will take a non-loopback + * device by default, if one is available after filtering directives + * + * NOTE: we accept MCA parameters, but info keys override them + */ +static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, + bool *need_listener) +{ + int flags = 0; + pmix_listener_t *lt; + int i, rc, saveindex = -1; + char **interfaces = NULL; + bool including = false; + char name[32]; + struct sockaddr_storage my_ss; + int kindex; + size_t n; + bool remote_connections = false; + bool tool_support = false; + pmix_socklen_t addrlen; + char *prefix, myhost[NI_MAXHOST+1]; + int myport; + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp setup_listener"); + + /* if we are not a server, then we shouldn't be doing this */ + if (PMIX_PROC_SERVER != pmix_globals.proc_type) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* scan the info keys and process any override instructions */ + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strcmp(info[n].key, PMIX_TCP_IF_INCLUDE)) { + if (NULL != mca_ptl_tcp_component.if_include) { + free(mca_ptl_tcp_component.if_include); + } + mca_ptl_tcp_component.if_include = strdup(info[n].value.data.string); + } else if (0 == strcmp(info[n].key, PMIX_TCP_IF_EXCLUDE)) { + if (NULL != mca_ptl_tcp_component.if_exclude) { + free(mca_ptl_tcp_component.if_exclude); + } + mca_ptl_tcp_component.if_exclude = strdup(info[n].value.data.string); + } else if (0 == strcmp(info[n].key, PMIX_TCP_IPV4_PORT)) { + mca_ptl_tcp_component.ipv4_port = info[n].value.data.integer; + } else if (0 == strcmp(info[n].key, PMIX_TCP_IPV6_PORT)) { + mca_ptl_tcp_component.ipv6_port = info[n].value.data.integer; + } else if (0 == strcmp(info[n].key, PMIX_TCP_DISABLE_IPV4)) { + if (PMIX_UNDEF == info[n].value.type) { + mca_ptl_tcp_component.disable_ipv4_family = true; + } else { + mca_ptl_tcp_component.disable_ipv4_family = info[n].value.data.flag; + } + } else if (0 == strcmp(info[n].key, PMIX_TCP_DISABLE_IPV6)) { + if (PMIX_UNDEF == info[n].value.type) { + mca_ptl_tcp_component.disable_ipv6_family = true; + } else { + mca_ptl_tcp_component.disable_ipv6_family = info[n].value.data.flag; + } + } else if (0 == strcmp(info[n].key, PMIX_SERVER_REMOTE_CONNECTIONS)) { + if (PMIX_UNDEF == info[n].value.type) { + remote_connections = true; + } else { + remote_connections = info[n].value.data.flag; + } + } else if (0 == strcmp(info[n].key, PMIX_TCP_URI)) { + if (NULL != mca_ptl_tcp_component.super.uri) { + free(mca_ptl_tcp_component.super.uri); + } + mca_ptl_tcp_component.super.uri = strdup(info[n].value.data.string); + } else if (0 == strcmp(info[n].key, PMIX_SYSTEM_TMPDIR)) { + if (NULL != mca_ptl_tcp_component.tmpdir) { + free(mca_ptl_tcp_component.tmpdir); + } + mca_ptl_tcp_component.tmpdir = strdup(info[n].value.data.string); + } else if (0 == strcmp(info[n].key, PMIX_SERVER_TOOL_SUPPORT)) { + if (PMIX_UNDEF == info[n].value.type) { + tool_support = true; + } else { + tool_support = info[n].value.data.flag; + } + } + } + } + + /* if interface include was given, construct a list + * of those interfaces which match the specifications - remember, + * the includes could be given as named interfaces, IP addrs, or + * subnet+mask + */ + if (NULL != mca_ptl_tcp_component.if_include) { + interfaces = split_and_resolve(&mca_ptl_tcp_component.if_include, + "include"); + including = true; + } else if (NULL != mca_ptl_tcp_component.if_exclude) { + interfaces = split_and_resolve(&mca_ptl_tcp_component.if_exclude, + "exclude"); + including = false; + } + + /* look at all available interfaces and pick one - we default to a + * loopback interface if available, but otherwise pick the first + * available interface since we are only talking locally */ + for (i = pmix_ifbegin(); i >= 0; i = pmix_ifnext(i)) { + if (PMIX_SUCCESS != pmix_ifindextoaddr(i, (struct sockaddr*)&my_ss, sizeof(my_ss))) { + pmix_output (0, "ptl_tcp: problems getting address for index %i (kernel index %i)\n", + i, pmix_ifindextokindex(i)); + continue; + } + /* ignore non-ip4/6 interfaces */ + if (AF_INET != my_ss.ss_family && + AF_INET6 != my_ss.ss_family) { + continue; + } + /* get the name for diagnostic purposes */ + pmix_ifindextoname(i, name, sizeof(name)); + + /* ignore any virtual interfaces */ + if (0 == strncmp(name, "vir", 3)) { + continue; + } + /* ignore any interfaces in a disabled family */ + if (AF_INET == my_ss.ss_family && + mca_ptl_tcp_component.disable_ipv4_family) { + continue; + } else if (AF_INET6 == my_ss.ss_family && + mca_ptl_tcp_component.disable_ipv6_family) { + continue; + } + /* get the kernel index */ + kindex = pmix_ifindextokindex(i); + if (kindex <= 0) { + continue; + } + pmix_output_verbose(10, pmix_ptl_base_framework.framework_output, + "WORKING INTERFACE %d KERNEL INDEX %d FAMILY: %s", i, kindex, + (AF_INET == my_ss.ss_family) ? "V4" : "V6"); + /* handle include/exclude directives */ + if (NULL != interfaces) { + /* check for match */ + rc = pmix_ifmatches(kindex, interfaces); + /* if one of the network specifications isn't parseable, then + * error out as we can't do what was requested + */ + if (PMIX_ERR_NETWORK_NOT_PARSEABLE == rc) { + pmix_show_help("help-ptl-tcp.txt", "not-parseable", true); + pmix_argv_free(interfaces); + return PMIX_ERR_BAD_PARAM; + } + /* if we are including, then ignore this if not present */ + if (including) { + if (PMIX_SUCCESS != rc) { + pmix_output_verbose(10, pmix_ptl_base_framework.framework_output, + "ptl:tcp:init rejecting interface %s (not in include list)", name); + continue; + } + } else { + /* we are excluding, so ignore if present */ + if (PMIX_SUCCESS == rc) { + pmix_output_verbose(10, pmix_ptl_base_framework.framework_output, + "ptl:tcp:init rejecting interface %s (in exclude list)", name); + continue; + } + } + } + + /* if this is the loopback device and they didn't enable + * remote connections, then we are done */ + if (pmix_ifisloopback(i)) { + if (remote_connections) { + /* ignore loopback */ + continue; + } else { + pmix_output_verbose(5, pmix_ptl_base_framework.framework_output, + "ptl:tcp:init loopback interface %s selected", name); + saveindex = i; + break; + } + } else { + /* if this is the first one we found, then hang on to it - we + * will use it if a loopback device is not found */ + if (saveindex < 0) { + saveindex = i; + } + } + } + /* cleanup */ + if (NULL != interfaces) { + pmix_argv_free(interfaces); + } + + /* if we didn't find anything, then we cannot operate */ + if (saveindex < 0) { + return PMIX_ERR_NOT_AVAILABLE; + } + + /* save the connection */ + if (PMIX_SUCCESS != pmix_ifindextoaddr(saveindex, + (struct sockaddr*)&mca_ptl_tcp_component.connection, + sizeof(struct sockaddr))) { + pmix_output (0, "ptl:tcp: problems getting address for kernel index %i\n", + pmix_ifindextokindex(saveindex)); + return PMIX_ERR_NOT_AVAILABLE; + } + + /* set the port */ + if (AF_INET == mca_ptl_tcp_component.connection.ss_family) { + ((struct sockaddr_in*) &mca_ptl_tcp_component.connection)->sin_port = htons(mca_ptl_tcp_component.ipv4_port); + if (0 != mca_ptl_tcp_component.ipv4_port) { + flags = 1; + } + } else if (AF_INET6 == mca_ptl_tcp_component.connection.ss_family) { + ((struct sockaddr_in6*) &mca_ptl_tcp_component.connection)->sin6_port = htons(mca_ptl_tcp_component.ipv6_port); + if (0 != mca_ptl_tcp_component.ipv6_port) { + flags = 1; + } + } + + lt = PMIX_NEW(pmix_listener_t); + lt->varname = strdup("PMIX_SERVER_URI2"); + lt->protocol = PMIX_PROTOCOL_V2; + lt->ptl = (struct pmix_ptl_module_t*)&pmix_ptl_tcp_module; + lt->cbfunc = connection_handler; + pmix_list_append(&pmix_ptl_globals.listeners, <->super); + + addrlen = sizeof(struct sockaddr_storage); + /* create a listen socket for incoming connection attempts */ + lt->socket = socket(mca_ptl_tcp_component.connection.ss_family, SOCK_STREAM, 0); + if (lt->socket < 0) { + printf("%s:%d socket() failed\n", __FILE__, __LINE__); + goto sockerror; + } + /* set reusing ports flag */ + if (setsockopt (lt->socket, SOL_SOCKET, SO_REUSEADDR, (const char *)&flags, sizeof(flags)) < 0) { + pmix_output(0, "ptl:tcp:create_listen: unable to set the " + "SO_REUSEADDR option (%s:%d)\n", + strerror(pmix_socket_errno), pmix_socket_errno); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + /* Set the socket to close-on-exec so that no children inherit + * this FD */ + if (pmix_fd_set_cloexec(lt->socket) != PMIX_SUCCESS) { + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + if (bind(lt->socket, (struct sockaddr*)&mca_ptl_tcp_component.connection, sizeof(struct sockaddr)) < 0) { + printf("%s:%d bind() failed: %s\n", __FILE__, __LINE__, strerror(errno)); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + /* resolve assigned port */ + if (getsockname(lt->socket, (struct sockaddr*)&mca_ptl_tcp_component.connection, &addrlen) < 0) { + pmix_output(0, "ptl:tcp:create_listen: getsockname(): %s (%d)", + strerror(pmix_socket_errno), pmix_socket_errno); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + /* setup listen backlog to maximum allowed by kernel */ + if (listen(lt->socket, SOMAXCONN) < 0) { + printf("%s:%d listen() failed\n", __FILE__, __LINE__); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + /* set socket up to be non-blocking, otherwise accept could block */ + if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) { + printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + flags |= O_NONBLOCK; + if (fcntl(lt->socket, F_SETFL, flags) < 0) { + printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + if (AF_INET == mca_ptl_tcp_component.connection.ss_family) { + prefix = "tcp4://"; + myport = ntohs(((struct sockaddr_in*) &mca_ptl_tcp_component.connection)->sin_port); + inet_ntop(AF_INET, &((struct sockaddr_in*) &mca_ptl_tcp_component.connection)->sin_addr, + myhost, NI_MAXHOST); + } else if (AF_INET6 == mca_ptl_tcp_component.connection.ss_family) { + prefix = "tcp6://"; + myport = ntohs(((struct sockaddr_in6*) &mca_ptl_tcp_component.connection)->sin6_port); + inet_ntop(AF_INET6, &((struct sockaddr_in6*) &mca_ptl_tcp_component.connection)->sin6_addr, + myhost, NI_MAXHOST); + } else { + goto sockerror; + } + + asprintf(<->uri, "%s.%d;%s%s:%d", pmix_globals.myid.nspace, pmix_globals.myid.rank, prefix, myhost, myport); + if (NULL == lt->uri) { + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp URI %s", lt->uri); + + /* if we are going to support tools, then drop the contact file */ + if (tool_support) { + FILE *fp; + + mca_ptl_tcp_component.filename = pmix_os_path(false, mca_ptl_tcp_component.tmpdir, "pmix-contact.txt", NULL); + if (NULL == mca_ptl_tcp_component.filename) { + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + fp = fopen(mca_ptl_tcp_component.filename, "w"); + if (NULL == fp) { + pmix_output( 0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.filename ); + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.filename); + mca_ptl_tcp_component.filename = NULL; + goto sockerror; + } + + /* output my nspace and rank plus the URI */ + fprintf(fp, "%s.%d:%s\n", pmix_globals.myid.nspace, pmix_globals.myid.rank, lt->uri); + fclose(fp); + } + + /* we need listener thread support */ + *need_listener = true; + + return PMIX_SUCCESS; + + sockerror: + pmix_list_remove_item(&pmix_ptl_globals.listeners, <->super); + PMIX_RELEASE(lt); + return PMIX_ERROR; +} + +/* + * Go through a list of argv; if there are any subnet specifications + * (a.b.c.d/e), resolve them to an interface name (Currently only + * supporting IPv4). If unresolvable, warn and remove. + */ +static char **split_and_resolve(char **orig_str, char *name) +{ + int i, ret, save, if_index; + char **argv, *str, *tmp; + char if_name[IF_NAMESIZE]; + struct sockaddr_storage argv_inaddr, if_inaddr; + uint32_t argv_prefix; + + /* Sanity check */ + if (NULL == orig_str || NULL == *orig_str) { + return NULL; + } + + argv = pmix_argv_split(*orig_str, ','); + if (NULL == argv) { + return NULL; + } + for (save = i = 0; NULL != argv[i]; ++i) { + if (isalpha(argv[i][0])) { + argv[save++] = argv[i]; + continue; + } + + /* Found a subnet notation. Convert it to an IP + address/netmask. Get the prefix first. */ + argv_prefix = 0; + tmp = strdup(argv[i]); + str = strchr(argv[i], '/'); + if (NULL == str) { + pmix_show_help("help-ptl-tcp.txt", "invalid if_inexclude", + true, name, tmp, "Invalid specification (missing \"/\")"); + free(argv[i]); + free(tmp); + continue; + } + *str = '\0'; + argv_prefix = atoi(str + 1); + + /* Now convert the IPv4 address */ + ((struct sockaddr*) &argv_inaddr)->sa_family = AF_INET; + ret = inet_pton(AF_INET, argv[i], + &((struct sockaddr_in*) &argv_inaddr)->sin_addr); + free(argv[i]); + + if (1 != ret) { + pmix_show_help("help-ptl-tcp.txt", "invalid if_inexclude", + true, name, tmp, + "Invalid specification (inet_pton() failed)"); + free(tmp); + continue; + } + pmix_output_verbose(20, pmix_ptl_base_framework.framework_output, + "ptl:tcp: Searching for %s address+prefix: %s / %u", + name, + pmix_net_get_hostname((struct sockaddr*) &argv_inaddr), + argv_prefix); + + /* Go through all interfaces and see if we can find a match */ + for (if_index = pmix_ifbegin(); if_index >= 0; + if_index = pmix_ifnext(if_index)) { + pmix_ifindextoaddr(if_index, + (struct sockaddr*) &if_inaddr, + sizeof(if_inaddr)); + if (pmix_net_samenetwork((struct sockaddr*) &argv_inaddr, + (struct sockaddr*) &if_inaddr, + argv_prefix)) { + break; + } + } + /* If we didn't find a match, keep trying */ + if (if_index < 0) { + pmix_show_help("help-ptl-tcp.txt", "invalid if_inexclude", + true, name, tmp, + "Did not find interface matching this subnet"); + free(tmp); + continue; + } + + /* We found a match; get the name and replace it in the + argv */ + pmix_ifindextoname(if_index, if_name, sizeof(if_name)); + pmix_output_verbose(20, pmix_ptl_base_framework.framework_output, + "ptl:tcp: Found match: %s (%s)", + pmix_net_get_hostname((struct sockaddr*) &if_inaddr), + if_name); + argv[save++] = strdup(if_name); + free(tmp); + } + + /* The list may have been compressed if there were invalid + entries, so ensure we end it with a NULL entry */ + argv[save] = NULL; + free(*orig_str); + *orig_str = pmix_argv_join(argv, ','); + return argv; +} + +static void connection_handler(int sd, short args, void *cbdata) +{ + pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata; + pmix_ptl_hdr_t hdr; + pmix_peer_t *peer; + pmix_rank_t rank; + pmix_status_t rc; + char *msg, *mg, *sec, *cred = NULL; + char *nspace; + uint32_t len, u32; + size_t cnt, msglen, n; + uint8_t flag; + pmix_nspace_t *nptr, *tmp; + bool found; + pmix_rank_info_t *info; + pmix_proc_t proc; + + pmix_output_verbose(8, pmix_ptl_base_framework.framework_output, + "ptl:tcp:connection_handler: new connection: %d", + pnd->sd); + + /* ensure the socket is in blocking mode */ + pmix_ptl_base_set_blocking(pnd->sd); + + /* ensure all is zero'd */ + memset(&hdr, 0, sizeof(pmix_ptl_hdr_t)); + + /* get the header */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(pnd->sd, (char*)&hdr, sizeof(pmix_ptl_hdr_t)))) { + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + /* get the id, authentication and version payload (and possibly + * security credential) - to guard against potential attacks, + * we'll set an arbitrary limit per a define */ + if (PMIX_MAX_CRED_SIZE < hdr.nbytes) { + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + if (NULL == (msg = (char*)malloc(hdr.nbytes))) { + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + if (PMIX_SUCCESS != pmix_ptl_base_recv_blocking(pnd->sd, msg, hdr.nbytes)) { + /* unable to complete the recv */ + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:tcp:connection_handler unable to complete recv of connect-ack with client ON SOCKET %d", + pnd->sd); + free(msg); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + cnt = hdr.nbytes; + mg = msg; + /* extract the name of the sec module they used */ + PMIX_STRNLEN(msglen, mg, cnt); + if (msglen < cnt) { + sec = msg; + mg += strlen(sec) + 1; + cnt -= strlen(sec) + 1; + } else { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + + /* extract any credential so we can validate this connection + * before doing anything else */ + if (sizeof(uint32_t) <= cnt) { + memcpy(&len, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + } else { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + /* convert it to host byte order */ + len = ntohl(len); + /* if a credential is present, then create space and + * extract it for processing */ + if (0 < len) { + cred = (char*)malloc(len); + if (NULL == cred) { + /* probably cannot send an error reply if we are out of memory */ + free(msg); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + memcpy(cred, mg, len); + mg += len; + cnt -= len; + } + + /* get the request type */ + if (1 <= cnt) { + memcpy(&flag, mg, 1); + ++mg; + --cnt; + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + + /* see if this is a tool connection request */ + if (1 == flag) { + /* does the server support tool connections? */ + if (NULL == pmix_host_server.tool_connected) { + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_SUPPORTED; + goto error; + } + /* extract the uid/gid */ + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->uid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + if (sizeof(uint32_t) <= cnt) { + memcpy(&u32, mg, sizeof(uint32_t)); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + pnd->gid = ntohl(u32); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + /* setup the info array to pass the relevant info + * to the server - starting with the version, if present */ + n = 0; + PMIX_STRNLEN(msglen, mg, cnt); + if (msglen < cnt) { + pnd->ninfo = 4; + PMIX_INFO_CREATE(pnd->info, pnd->ninfo); + (void)strncpy(pnd->info[n].key, PMIX_VERSION_INFO, PMIX_MAX_KEYLEN); + pnd->info[n].value.type = PMIX_STRING; + pnd->info[n].value.data.string = strdup(mg); + ++n; + } else { + pnd->ninfo = 3; + PMIX_INFO_CREATE(pnd->info, pnd->ninfo); + } + /* provide the user id */ + (void)strncpy(pnd->info[n].key, PMIX_USERID, PMIX_MAX_KEYLEN); + pnd->info[n].value.type = PMIX_UINT32; + pnd->info[n].value.data.uint32 = pnd->uid; + ++n; + /* and the group id */ + (void)strncpy(pnd->info[n].key, PMIX_GRPID, PMIX_MAX_KEYLEN); + pnd->info[n].value.type = PMIX_UINT32; + pnd->info[n].value.data.uint32 = pnd->gid; + ++n; + /* pass along the bfrop, buffer_type, and sec fields so + * we can assign them once we create a peer object */ + if (NULL != sec) { + pnd->psec = strdup(sec); + } + /* pass along the credential for later */ + if (NULL != cred) { + pnd->cred = cred; + pnd->len = len; + } + /* release the msg */ + free(msg); + /* request an nspace for this requestor - it will + * automatically be assigned rank=0 */ + pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd); + return; + } + + /* they must be a client, so get their nspace/rank */ + PMIX_STRNLEN(msglen, mg, cnt); + if (msglen < cnt) { + nspace = mg; + mg += strlen(nspace) + 1; + cnt -= strlen(nspace) + 1; + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + + if (sizeof(pmix_rank_t) <= cnt) { + /* have to convert this to host order */ + memcpy(&u32, mg, sizeof(uint32_t)); + rank = ntohl(u32); + mg += sizeof(uint32_t); + cnt -= sizeof(uint32_t); + } else { + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_BAD_PARAM; + goto error; + } + + /* see if we know this nspace */ + nptr = NULL; + PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(tmp->nspace, nspace)) { + nptr = tmp; + break; + } + } + if (NULL == nptr) { + /* we don't know this namespace, reject it */ + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + + /* see if we have this peer in our list */ + info = NULL; + found = false; + PMIX_LIST_FOREACH(info, &nptr->server->ranks, pmix_rank_info_t) { + if (info->rank == rank) { + found = true; + break; + } + } + if (!found) { + /* rank unknown, reject it */ + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + + /* a peer can connect on multiple sockets since it can fork/exec + * a child that also calls PMIX_Init, so add it here if necessary. + * Create the tracker for this peer */ + peer = PMIX_NEW(pmix_peer_t); + if (NULL == peer) { + /* probably cannot send an error reply if we are out of memory */ + free(msg); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + PMIX_RETAIN(info); + peer->info = info; + info->proc_cnt++; /* increase number of processes on this rank */ + peer->sd = pnd->sd; + if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) { + free(msg); + info->proc_cnt--; + PMIX_RELEASE(info); + PMIX_RELEASE(peer); + /* probably cannot send an error reply if we are out of memory */ + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + /* set the sec module to match this peer */ + if (PMIX_SUCCESS != (rc = pmix_psec.assign_module(peer, sec))) { + free(msg); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); + PMIX_RELEASE(peer); + /* send an error reply to the client */ + goto error; + } + free(msg); + + /* the choice of PTL module is obviously us */ + peer->compat.ptl = &pmix_ptl_tcp_module; + + /* validate the connection */ + if (PMIX_SUCCESS != (rc = pmix_psec.validate_connection((struct pmix_peer_t*)peer, + PMIX_PROTOCOL_V2, cred, len))) { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "validation of client connection failed"); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); + PMIX_RELEASE(peer); + /* send an error reply to the client */ + goto error; + } + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "client connection validated"); + + /* tell the client all is good */ + u32 = htonl(PMIX_SUCCESS); + if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t))) { + PMIX_ERROR_LOG(rc); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); + PMIX_RELEASE(peer); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + /* send the client's array index */ + u32 = htonl(peer->index); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); + PMIX_RELEASE(peer); + goto error; + } + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "connect-ack from client completed"); + + /* let the host server know that this client has connected */ + if (NULL != pmix_host_server.client_connected) { + (void)strncpy(proc.nspace, peer->info->nptr->nspace, PMIX_MAX_NSLEN); + proc.rank = peer->info->rank; + rc = pmix_host_server.client_connected(&proc, peer->info->server_object, + NULL, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } + + pmix_ptl_base_set_nonblocking(pnd->sd); + + /* start the events for this client */ + event_assign(&peer->recv_event, pmix_globals.evbase, pnd->sd, + EV_READ|EV_PERSIST, pmix_ptl_base_recv_handler, peer); + event_add(&peer->recv_event, NULL); + peer->recv_ev_active = true; + event_assign(&peer->send_event, pmix_globals.evbase, pnd->sd, + EV_WRITE|EV_PERSIST, pmix_ptl_base_send_handler, peer); + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "pmix:server client %s:%u has connected on socket %d", + peer->info->nptr->nspace, peer->info->rank, peer->sd); + PMIX_RELEASE(pnd); + return; + + error: + /* send an error reply to the client */ + if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(pnd->sd, (char*)&rc, sizeof(int))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + } + PMIX_RELEASE(pnd); + return; +} + +/* process the callback with tool connection info */ +static void process_cbfunc(int sd, short args, void *cbdata) +{ + pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; + pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cd->cbdata; + pmix_nspace_t *nptr; + pmix_rank_info_t *info; + int rc; + uint32_t u32; + + /* send this status so they don't hang */ + u32 = ntohl(cd->status); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } + + /* if the request failed, then we are done */ + if (PMIX_SUCCESS != cd->status) { + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } + + /* send the nspace back to the tool */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } + + /* send my nspace back to the tool */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } + + /* send my rank back to the tool */ + u32 = ntohl(pmix_globals.myid.rank); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + return; + } + + /* add this nspace to our pool */ + nptr = PMIX_NEW(pmix_nspace_t); + (void)strncpy(nptr->nspace, cd->proc.nspace, PMIX_MAX_NSLEN); + nptr->server = PMIX_NEW(pmix_server_nspace_t); + pmix_list_append(&pmix_globals.nspaces, &nptr->super); + /* add this tool rank to the nspace */ + info = PMIX_NEW(pmix_rank_info_t); + PMIX_RETAIN(nptr); + info->nptr = nptr; + info->rank = 0; + /* need to include the uid/gid for validation */ + info->uid = pnd->uid; + info->gid = pnd->gid; + pmix_list_append(&nptr->server->ranks, &info->super); + + /* setup a peer object for this tool */ + pmix_peer_t *peer = PMIX_NEW(pmix_peer_t); + PMIX_RETAIN(info); + peer->info = info; + peer->proc_cnt = 1; + peer->sd = pnd->sd; + + /* get the appropriate compatibility modules based on the + * info provided by the tool during the initial connection request */ + if (PMIX_SUCCESS != pmix_psec.assign_module((struct pmix_peer_t*)peer, pnd->psec)) { + PMIX_RELEASE(peer); + pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); + PMIX_RELEASE(nptr); // will release the info object + CLOSE_THE_SOCKET(pnd->sd); + goto done; + } + /* the choice of PTL module was obviously made by the connecting + * tool as we received this request via that channel, so simply + * record it here for future use */ + peer->compat.ptl = &pmix_ptl_tcp_module; + + /* validate the connection */ + if (PMIX_SUCCESS != (rc = pmix_psec.validate_connection((struct pmix_peer_t*)peer, + PMIX_PROTOCOL_V2, + pnd->cred, pnd->len))) { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "validation of tool credentials failed: %s", + PMIx_Error_string(rc)); + PMIX_RELEASE(peer); + pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); + PMIX_RELEASE(nptr); // will release the info object + CLOSE_THE_SOCKET(pnd->sd); + goto done; + } + + /* set the socket non-blocking for all further operations */ + pmix_ptl_base_set_nonblocking(pnd->sd); + + if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) { + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); + PMIX_RELEASE(peer); + pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); + PMIX_RELEASE(nptr); // will release the info object + /* probably cannot send an error reply if we are out of memory */ + return; + } + + /* start the events for this tool */ + event_assign(&peer->recv_event, pmix_globals.evbase, pnd->sd, + EV_READ|EV_PERSIST, pmix_ptl_base_recv_handler, peer); + event_add(&peer->recv_event, NULL); + peer->recv_ev_active = true; + event_assign(&peer->send_event, pmix_globals.evbase, pnd->sd, + EV_WRITE|EV_PERSIST, pmix_ptl_base_send_handler, peer); + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "pmix:server tool %s:%d has connected on socket %d", + peer->info->nptr->nspace, peer->info->rank, peer->sd); + + done: + PMIX_RELEASE(pnd); + PMIX_RELEASE(cd); +} + +/* receive a callback from the host RM with an nspace + * for a connecting tool */ +static void cnct_cbfunc(pmix_status_t status, + pmix_proc_t *proc, void *cbdata) +{ + pmix_setup_caddy_t *cd; + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "pmix:tcp:cnct_cbfunc returning %s:%d", + proc->nspace, proc->rank); + + /* need to thread-shift this into our context */ + cd = PMIX_NEW(pmix_setup_caddy_t); + cd->status = status; + (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); + cd->cbdata = cbdata; + PMIX_THREADSHIFT(cd, process_cbfunc); +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am new file mode 100644 index 0000000000..e6606e2e84 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am @@ -0,0 +1,50 @@ +# -*- makefile -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. +# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers = ptl_usock.h +sources = \ + ptl_usock_component.c \ + ptl_usock.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_ptl_usock_DSO +lib = +lib_sources = +component = mca_ptl_usock.la +component_sources = $(headers) $(sources) +else +lib = libmca_ptl_usock.la +lib_sources = $(headers) $(sources) +component = +component_sources = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component) +mca_ptl_usock_la_SOURCES = $(component_sources) +mca_ptl_usock_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(lib) +libmca_ptl_usock_la_SOURCES = $(lib_sources) +libmca_ptl_usock_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c new file mode 100644 index 0000000000..4a34896d08 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c @@ -0,0 +1,358 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include "pmix_common.h" + +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_SYS_UN_H +#include +#endif +#ifdef HAVE_SYS_UIO_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif + +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/client/pmix_client_ops.h" +#include "src/include/pmix_globals.h" +#include "src/include/pmix_socket_errno.h" +#include "src/mca/psec/psec.h" + +#include "src/mca/ptl/base/base.h" +#include "ptl_usock.h" + +static pmix_status_t init(void); +static void finalize(void); +static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, + pmix_info_t *info, size_t ninfo); +static pmix_status_t send_recv(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_cbfunc_t cbfunc, + void *cbdata); +static pmix_status_t send_oneway(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_tag_t tag); + +pmix_ptl_module_t pmix_ptl_usock_module = { + .init = init, + .finalize = finalize, + .send_recv = send_recv, + .send = send_oneway, + .connect_to_peer = connect_to_peer +}; + +static pmix_status_t recv_connect_ack(int sd); +static pmix_status_t send_connect_ack(int sd); + +static pmix_status_t init(void) +{ + return PMIX_SUCCESS; +} + +static void finalize(void) +{ +} + +static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, + pmix_info_t *info, size_t ninfo) +{ + struct sockaddr_un *address; + char *evar, **uri; + pmix_status_t rc; + int sd; + pmix_socklen_t len; + + /* if we are not a client, there is nothing we can do */ + if (!PMIX_PROC_IS_CLIENT) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* if we don't have a path to the daemon rendezvous point, + * then we need to return an error */ + if (NULL == (evar = getenv("PMIX_SERVER_URI"))) { + /* let the caller know that the server isn't available */ + return PMIX_ERR_SERVER_NOT_AVAIL; + } + uri = pmix_argv_split(evar, ':'); + if (3 != pmix_argv_count(uri)) { + pmix_argv_free(uri); + return PMIX_ERROR; + } + + /* set the server nspace */ + pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t); + pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t); + (void)strncpy(pmix_client_globals.myserver.info->nptr->nspace, uri[0], PMIX_MAX_NSLEN); + + /* set the server rank */ + pmix_client_globals.myserver.info->rank = strtoull(uri[1], NULL, 10); + + /* setup the path to the daemon rendezvous point */ + memset(&mca_ptl_usock_component.connection, 0, sizeof(struct sockaddr_storage)); + address = (struct sockaddr_un*)&mca_ptl_usock_component.connection; + address->sun_family = AF_UNIX; + snprintf(address->sun_path, sizeof(address->sun_path)-1, "%s", uri[2]); + /* if the rendezvous file doesn't exist, that's an error */ + if (0 != access(uri[2], R_OK)) { + pmix_argv_free(uri); + return PMIX_ERR_NOT_FOUND; + } + pmix_argv_free(uri); + + /* establish the connection */ + len = sizeof(struct sockaddr_un); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&mca_ptl_usock_component.connection, len, &sd))) { + PMIX_ERROR_LOG(rc); + return rc; + } + pmix_client_globals.myserver.sd = sd; + + /* send our identity and any authentication credentials to the server */ + if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) { + CLOSE_THE_SOCKET(sd); + return rc; + } + + /* do whatever handshake is required */ + if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) { + CLOSE_THE_SOCKET(sd); + return rc; + } + + pmix_output_verbose(2, pmix_globals.debug_output, + "sock_peer_try_connect: Connection across to server succeeded"); + + /* mark the connection as made */ + pmix_globals.connected = true; + + pmix_ptl_base_set_nonblocking(sd); + + /* setup recv event */ + event_assign(&pmix_client_globals.myserver.recv_event, + pmix_globals.evbase, + pmix_client_globals.myserver.sd, + EV_READ | EV_PERSIST, + pmix_ptl_base_recv_handler, &pmix_client_globals.myserver); + event_add(&pmix_client_globals.myserver.recv_event, 0); + pmix_client_globals.myserver.recv_ev_active = true; + + /* setup send event */ + event_assign(&pmix_client_globals.myserver.send_event, + pmix_globals.evbase, + pmix_client_globals.myserver.sd, + EV_WRITE|EV_PERSIST, + pmix_ptl_base_send_handler, &pmix_client_globals.myserver); + pmix_client_globals.myserver.send_ev_active = false; + + return PMIX_SUCCESS; +} + +static pmix_status_t send_recv(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_cbfunc_t cbfunc, + void *cbdata) +{ + pmix_ptl_sr_t *ms; + pmix_output_verbose(5, pmix_globals.debug_output, + "[%s:%d] post send to server", + __FILE__, __LINE__); + ms = PMIX_NEW(pmix_ptl_sr_t); + ms->peer = peer; + ms->bfr = bfr; + ms->cbfunc = cbfunc; + ms->cbdata = cbdata; + event_assign(&ms->ev, pmix_globals.evbase, -1, + EV_WRITE, pmix_ptl_base_send_recv, ms); + event_active(&ms->ev, EV_WRITE, 1); + return PMIX_SUCCESS; +} + +static pmix_status_t send_oneway(struct pmix_peer_t *peer, + pmix_buffer_t *bfr, + pmix_ptl_tag_t tag) +{ + pmix_ptl_queue_t *q; + + /* we have to transfer this to an event for thread + * safety as we need to post this message on the + * peer's send queue */ + q = PMIX_NEW(pmix_ptl_queue_t); + q->peer = peer; + q->buf = bfr; + q->tag = tag; + event_assign(&q->ev, pmix_globals.evbase, -1, + EV_WRITE, pmix_ptl_base_send, q); + event_active(&q->ev, EV_WRITE, 1); + + return PMIX_SUCCESS; +} + +static pmix_status_t send_connect_ack(int sd) +{ + char *msg; + pmix_usock_hdr_t hdr; + size_t sdsize=0, csize=0, len; + char *cred = NULL; + pmix_status_t rc; + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: SEND CONNECT ACK"); + + /* setup the header */ + memset(&hdr, 0, sizeof(pmix_usock_hdr_t)); + hdr.pindex = -1; + hdr.tag = UINT32_MAX; + + /* reserve space for the nspace and rank info */ + sdsize = strlen(pmix_globals.myid.nspace) + 1 + sizeof(int); + + /* get a credential, if the security system provides one. Not + * every SPC will do so, thus we must first check */ + if (PMIX_SUCCESS != (rc = pmix_psec.create_cred(&pmix_client_globals.myserver, + PMIX_PROTOCOL_V1, &cred, &len))) { + return rc; + } + + /* set the number of bytes to be read beyond the header */ + hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + len; // must NULL terminate the VERSION string! + + /* create a space for our message */ + sdsize = (sizeof(hdr) + hdr.nbytes); + if (NULL == (msg = (char*)malloc(sdsize))) { + if (NULL != cred) { + free(cred); + } + return PMIX_ERR_OUT_OF_RESOURCE; + } + memset(msg, 0, sdsize); + + /* load the message */ + csize=0; + memcpy(msg, &hdr, sizeof(pmix_usock_hdr_t)); + csize += sizeof(pmix_usock_hdr_t); + memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace)); + csize += strlen(pmix_globals.myid.nspace)+1; + memcpy(msg+csize, &pmix_globals.myid.rank, sizeof(int)); + csize += sizeof(int); + memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION)); + csize += strlen(PMIX_VERSION)+1; + if (NULL != cred) { + memcpy(msg+csize, cred, strlen(cred)); // leaves last position in msg set to NULL + } + + if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) { + free(msg); + if (NULL != cred) { + free(cred); + } + return PMIX_ERR_UNREACH; + } + free(msg); + if (NULL != cred) { + free(cred); + } + return PMIX_SUCCESS; +} + +/* we receive a connection acknowledgement from the server, + * consisting of nothing more than a status report. If success, + * then we initiate authentication method */ +static pmix_status_t recv_connect_ack(int sd) +{ + pmix_status_t reply; + pmix_status_t rc; + struct timeval tv, save; + pmix_socklen_t sz; + bool sockopt = true; + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: RECV CONNECT ACK FROM SERVER"); + + /* get the current timeout value so we can reset to it */ + sz = sizeof(save); + if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) { + if (ENOPROTOOPT == errno) { + sockopt = false; + } else { + return PMIX_ERR_UNREACH; + } + } else { + /* set a timeout on the blocking recv so we don't hang */ + tv.tv_sec = 2; + tv.tv_usec = 0; + if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); + return PMIX_ERR_UNREACH; + } + } + + /* receive the status reply */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&reply, sizeof(int)); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + + /* see if they want us to do the handshake */ + if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { + if (PMIX_SUCCESS != (rc = pmix_psec.client_handshake(&pmix_client_globals.myserver, sd))) { + return rc; + } + } else if (PMIX_SUCCESS != reply) { + return reply; + } + + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: RECV CONNECT CONFIRMATION"); + + /* receive our index into the server's client array */ + rc = pmix_ptl_base_recv_blocking(sd, (char*)&pmix_globals.pindex, sizeof(int)); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + return rc; + } + if (sockopt) { + /* return the socket to normal */ + if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) { + return PMIX_ERR_UNREACH; + } + } + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.h b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.h new file mode 100644 index 0000000000..358b23ab37 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_PTL_USOCK_H +#define PMIX_PTL_USOCK_H + +#include "src/mca/ptl/ptl.h" + +BEGIN_C_DECLS + +typedef struct { + pmix_ptl_base_component_t super; + char *tmpdir; + struct sockaddr_storage connection; + char *filename; +} pmix_ptl_usock_component_t; + +/* header for messages */ +typedef struct { + int pindex; + uint32_t tag; + size_t nbytes; +} pmix_usock_hdr_t; + +extern pmix_ptl_usock_component_t mca_ptl_usock_component; + +extern pmix_ptl_module_t pmix_ptl_usock_module; + +END_C_DECLS + +#endif /* PMIX_PTL_USOCK_H */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c new file mode 100644 index 0000000000..6eba2a42d5 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c @@ -0,0 +1,621 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennptlee and The University + * of Tennptlee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include +#include + +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_SYS_UN_H +#include +#endif +#ifdef HAVE_SYS_UIO_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#include + +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/fd.h" +#include "src/util/show_help.h" +#include "src/util/strnlen.h" +#include "src/mca/psec/psec.h" +#include "src/server/pmix_server_ops.h" + +#include "src/mca/ptl/base/base.h" +#include "src/mca/ptl/usock/ptl_usock.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static int component_query(pmix_mca_base_module_t **module, int *priority); +static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, + bool *need_listener); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ + pmix_ptl_usock_component_t mca_ptl_usock_component = { + .super = { + .base = { + PMIX_PTL_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "usock", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query + }, + .priority = 5, + .uri = NULL, + .setup_listener = setup_listener + }, + .tmpdir = NULL, + .filename = NULL +}; + +static void connection_handler(int sd, short args, void *cbdata); +static void listener_cb(int incoming_sd, void *cbdata); +static char *sec_mode = NULL; + +pmix_status_t component_open(void) +{ + char *tdir; + + memset(&mca_ptl_usock_component.connection, 0, sizeof(mca_ptl_usock_component.connection)); + + /* check for environ-based directives + * on system tmpdir to use */ + if (NULL == (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) { + if (NULL == (tdir = getenv("TMPDIR"))) { + if (NULL == (tdir = getenv("TEMP"))) { + if (NULL == (tdir = getenv("TMP"))) { + tdir = "/tmp"; + } + } + } + } + if (NULL != tdir) { + mca_ptl_usock_component.tmpdir = strdup(tdir); + } + + return PMIX_SUCCESS; +} + + +pmix_status_t component_close(void) +{ + if (NULL != sec_mode) { + free(sec_mode); + } + if (NULL != mca_ptl_usock_component.tmpdir) { + free(mca_ptl_usock_component.tmpdir); + } + if (NULL != mca_ptl_usock_component.super.uri) { + free(mca_ptl_usock_component.super.uri); + } + if (NULL != mca_ptl_usock_component.filename) { + /* remove the file */ + unlink(mca_ptl_usock_component.filename); + free(mca_ptl_usock_component.filename); + } + + return PMIX_SUCCESS; +} + +static int component_query(pmix_mca_base_module_t **module, int *priority) +{ + *module = (pmix_mca_base_module_t*)&pmix_ptl_usock_module; + return PMIX_SUCCESS; +} + +/* if we are the server, then we need to setup a usock rendezvous + * point for legacy releases, but only do so if requested as some + * systems may not wish to support older releases. The system can, + * of course, simply use the MCA param method to disable this + * component (PMIX_MCA_ptl=^usock), or can tell us to disqualify + * ourselves using an info key to this API. + * + * NOTE: we accept MCA parameters, but info keys override them + */ +static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, + bool *need_listener) +{ + int flags; + size_t n; + pmix_listener_t *lt; + pmix_status_t rc; + socklen_t addrlen; + struct sockaddr_un *address; + bool disabled = false; + char *secmods, **options, *pmix_pid; + pid_t mypid; + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:usock setup_listener"); + + /* if we are not a server, then we shouldn't be doing this */ + if (PMIX_PROC_SERVER != pmix_globals.proc_type) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* scan the info keys and process any override instructions */ + if (NULL != info) { + for (n=0; n < ninfo; n++) { + if (0 == strcmp(info[n].key, PMIX_USOCK_DISABLE)) { + if (PMIX_UNDEF == info[n].value.type) { + disabled = true;; + } else { + disabled = info[n].value.data.flag; + } + break; + } + } + } + + /* see if we have been disabled */ + if (disabled) { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:usock not available"); + return PMIX_ERR_NOT_AVAILABLE; + } + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "ptl:usock setting up listener"); + + addrlen = sizeof(struct sockaddr_un); + address = (struct sockaddr_un*)&mca_ptl_usock_component.connection; + address->sun_family = AF_UNIX; + + /* any client we hear from will be using v1.x protocols. This + * means that they cannot tell us what security module they + * are using as this wasn't included in their handshake. So + * the best we can assume is that they are using the highest + * priority default we have */ + secmods = pmix_psec.get_available_modules(); + options = pmix_argv_split(secmods, ','); + sec_mode = strdup(options[0]); + pmix_argv_free(options); + free(secmods); + + /* define the listener */ + lt = PMIX_NEW(pmix_listener_t); + + /* for now, just setup the v1.1 series rendezvous point + * we use the pid to reduce collisions */ + mypid = getpid(); + if (0 > asprintf(&pmix_pid, "%s/pmix-%d", mca_ptl_usock_component.tmpdir, mypid)) { + PMIX_RELEASE(lt); + return PMIX_ERR_NOMEM; + } + if ((strlen(pmix_pid) + 1) > sizeof(address->sun_path)-1) { + pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true, + mca_ptl_usock_component.tmpdir, pmix_pid); + free(pmix_pid); + PMIX_RELEASE(lt); + return PMIX_ERR_INVALID_LENGTH; + } + snprintf(address->sun_path, sizeof(address->sun_path)-1, "%s", pmix_pid); + free(pmix_pid); + /* set the URI */ + lt->varname = strdup("PMIX_SERVER_URI"); + if (0 > asprintf(<->uri, "%s:%lu:%s", pmix_globals.myid.nspace, + (unsigned long)pmix_globals.myid.rank, address->sun_path)) { + PMIX_RELEASE(lt); + return PMIX_ERR_NOMEM; + } + /* save the rendezvous filename for later removal */ + mca_ptl_usock_component.filename = strdup(address->sun_path); + + lt->protocol = PMIX_PROTOCOL_V1; + lt->ptl = (struct pmix_ptl_module_t*)&pmix_ptl_usock_module; + lt->cbfunc = connection_handler; + pmix_list_append(&pmix_ptl_globals.listeners, <->super); + + /* create a listen socket for incoming connection attempts */ + lt->socket = socket(PF_UNIX, SOCK_STREAM, 0); + if (lt->socket < 0) { + printf("%s:%d socket() failed\n", __FILE__, __LINE__); + goto sockerror; + } + /* Set the socket to close-on-exec so that no children inherit + * this FD */ + if (pmix_fd_set_cloexec(lt->socket) != PMIX_SUCCESS) { + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + if (bind(lt->socket, (struct sockaddr*)address, addrlen) < 0) { + printf("%s:%d bind() failed\n", __FILE__, __LINE__); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + /* chown as required */ + if (lt->owner_given) { + if (0 != chown(address->sun_path, lt->owner, -1)) { + pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno)); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + } + if (lt->group_given) { + if (0 != chown(address->sun_path, -1, lt->group)) { + pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno)); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + } + /* set the mode as required */ + if (0 != chmod(address->sun_path, lt->mode)) { + pmix_output(0, "CANNOT CHMOD socket %s: %s", address->sun_path, strerror (errno)); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + /* setup listen backlog to maximum allowed by kernel */ + if (listen(lt->socket, SOMAXCONN) < 0) { + printf("%s:%d listen() failed\n", __FILE__, __LINE__); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + /* set socket up to be non-blocking, otherwise accept could block */ + if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) { + printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + flags |= O_NONBLOCK; + if (fcntl(lt->socket, F_SETFL, flags) < 0) { + printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__); + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + + /* if the server will listen for us, then ask it to do so now */ + rc = PMIX_ERR_NOT_SUPPORTED; + if (NULL != pmix_host_server.listener) { + rc = pmix_host_server.listener(lt->socket, listener_cb, (void*)lt); + } + + if (PMIX_SUCCESS != rc) { + *need_listener = true; + } + + return PMIX_SUCCESS; + + sockerror: + pmix_list_remove_item(&pmix_ptl_globals.listeners, <->super); + PMIX_RELEASE(lt); + return PMIX_ERROR; +} + +static void listener_cb(int incoming_sd, void *cbdata) +{ + pmix_pending_connection_t *pending_connection; + + /* throw it into our event library for processing */ + pmix_output_verbose(8, pmix_ptl_base_framework.framework_output, + "listen_cb: pushing new connection %d into evbase", + incoming_sd); + pending_connection = PMIX_NEW(pmix_pending_connection_t); + pending_connection->sd = incoming_sd; + event_assign(&pending_connection->ev, pmix_globals.evbase, -1, + EV_WRITE, connection_handler, pending_connection); + event_active(&pending_connection->ev, EV_WRITE, 1); +} + +/* Parse init-ack message: + * NSPACE<0>VERSION<0>[CRED<0>] + */ +static pmix_status_t parse_connect_ack (char *msg, unsigned int len, + char **nspace, unsigned int *rank, + char **version, char **cred) +{ + unsigned int msglen; + + PMIX_STRNLEN(msglen, msg, len); + if (msglen < len) { + *nspace = msg; + msg += strlen(*nspace) + 1; + len -= strlen(*nspace) + 1; + } else { + return PMIX_ERR_BAD_PARAM; + } + + PMIX_STRNLEN(msglen, msg, len); + if (msglen <= len) { + memcpy(rank, msg, sizeof(int)); + msg += sizeof(int); + len -= sizeof(int); + } else { + return PMIX_ERR_BAD_PARAM; + } + + PMIX_STRNLEN(msglen, msg, len); + if (msglen < len) { + *version = msg; + msg += strlen(*version) + 1; + len -= strlen(*version) + 1; + } else { + return PMIX_ERR_BAD_PARAM; + } + + PMIX_STRNLEN(msglen, msg, len); + if (msglen < len) + *cred = msg; + else { + *cred = NULL; + } + + return PMIX_SUCCESS; +} + + +static void connection_handler(int sd, short args, void *cbdata) +{ + pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata; + char *msg, *nspace, *version, *cred; + pmix_status_t rc; + unsigned int rank; + pmix_usock_hdr_t hdr; + pmix_nspace_t *nptr, *tmp; + pmix_rank_info_t *info; + pmix_peer_t *psave = NULL; + bool found; + pmix_proc_t proc; + size_t len; + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "USOCK CONNECTION FROM PEER ON SOCKET %d", pnd->sd); + + /* ensure all is zero'd */ + memset(&hdr, 0, sizeof(pmix_usock_hdr_t)); + + /* get the header */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(pnd->sd, (char*)&hdr, sizeof(pmix_usock_hdr_t)))) { + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + /* get the id, authentication and version payload (and possibly + * security credential) - to guard against potential attacks, + * we'll set an arbitrary limit per a define */ + if (PMIX_MAX_CRED_SIZE < hdr.nbytes) { + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + if (NULL == (msg = (char*)malloc(hdr.nbytes))) { + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + if (PMIX_SUCCESS != pmix_ptl_base_recv_blocking(pnd->sd, msg, hdr.nbytes)) { + /* unable to complete the recv */ + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "unable to complete recv of connect-ack with client ON SOCKET %d", pnd->sd); + free(msg); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + if (PMIX_SUCCESS != (rc = parse_connect_ack (msg, hdr.nbytes, &nspace, + &rank, &version, &cred))) { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "error parsing connect-ack from client ON SOCKET %d", pnd->sd); + free(msg); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "connect-ack recvd from peer %s:%d:%s on socket %d", + nspace, rank, version, pnd->sd); + + /* do not check the version - we only retain it at this + * time in case we need to check it at some future date. + * For now, our intent is to retain backward compatibility + * and so we will assume that all versions are compatible. */ + + /* see if we know this nspace */ + nptr = NULL; + PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_nspace_t) { + if (0 == strcmp(tmp->nspace, nspace)) { + nptr = tmp; + break; + } + } + if (NULL == nptr) { + /* we don't know this namespace, reject it */ + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + + /* see if we have this peer in our list */ + info = NULL; + found = false; + PMIX_LIST_FOREACH(info, &nptr->server->ranks, pmix_rank_info_t) { + if (info->rank == rank) { + found = true; + break; + } + } + if (!found) { + /* rank unknown, reject it */ + free(msg); + /* send an error reply to the client */ + rc = PMIX_ERR_NOT_FOUND; + goto error; + } + /* a peer can connect on multiple sockets since it can fork/exec + * a child that also calls PMIx_Init, so add it here if necessary. + * Create the tracker for this peer */ + psave = PMIX_NEW(pmix_peer_t); + if (NULL == psave) { + rc = PMIX_ERR_NOMEM; + goto error; + } + PMIX_RETAIN(info); + psave->info = info; + info->proc_cnt++; /* increase number of processes on this rank */ + psave->sd = pnd->sd; + if (0 > (psave->index = pmix_pointer_array_add(&pmix_server_globals.clients, psave))) { + free(msg); + info->proc_cnt--; + PMIX_RELEASE(info); + PMIX_RELEASE(psave); + /* probably cannot send an error reply if we are out of memory */ + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + /* get the appropriate compatibility modules */ + if (PMIX_SUCCESS != pmix_psec.assign_module((struct pmix_peer_t*)psave, sec_mode)) { + info->proc_cnt--; + PMIX_RELEASE(info); + PMIX_RELEASE(psave); + pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); + /* send an error reply to the client */ + goto error; + } + /* the choice of PTL module was obviously made by the connecting + * tool as we received this request via that channel, so simply + * record it here for future use */ + psave->compat.ptl = &pmix_ptl_usock_module; + + /* validate the connection */ + if (NULL == cred) { + len = 0; + } else { + len = strlen(cred); + } + if (PMIX_SUCCESS != (rc = pmix_psec.validate_connection((struct pmix_peer_t*)psave, + PMIX_PROTOCOL_V1, cred, len))) { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "validation of client credentials failed: %s", + PMIx_Error_string(rc)); + info->proc_cnt--; + PMIX_RELEASE(info); + PMIX_RELEASE(psave); + pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); + /* send an error reply to the client */ + goto error; + } + + /* send them success */ + rc = PMIX_SUCCESS; + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) { + PMIX_ERROR_LOG(rc); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); + PMIX_RELEASE(psave); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + /* send the client's array index */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) { + PMIX_ERROR_LOG(rc); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); + PMIX_RELEASE(psave); + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "connect-ack from client completed"); + + /* let the host server know that this client has connected */ + if (NULL != pmix_host_server.client_connected) { + (void)strncpy(proc.nspace, psave->info->nptr->nspace, PMIX_MAX_NSLEN); + proc.rank = psave->info->rank; + rc = pmix_host_server.client_connected(&proc, psave->info->server_object, NULL, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); + PMIX_RELEASE(psave); + CLOSE_THE_SOCKET(pnd->sd); + } + } + + /* start the events for this client */ + event_assign(&psave->recv_event, pmix_globals.evbase, pnd->sd, + EV_READ|EV_PERSIST, pmix_ptl_base_recv_handler, psave); + event_add(&psave->recv_event, NULL); + psave->recv_ev_active = true; + event_assign(&psave->send_event, pmix_globals.evbase, pnd->sd, + EV_WRITE|EV_PERSIST, pmix_ptl_base_send_handler, psave); + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "pmix:server client %s:%u has connected on socket %d", + psave->info->nptr->nspace, psave->info->rank, psave->sd); + + PMIX_RELEASE(pnd); + return; + + error: + /* send an error reply to the client */ + if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(pnd->sd, (char*)&rc, sizeof(int))) { + PMIX_ERROR_LOG(rc); + } + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c index 5a9f38afa4..53ed907db2 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c +++ b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c @@ -28,7 +28,6 @@ #include "src/class/pmix_object.h" #include "src/client/pmix_client_ops.h" -#include "src/usock/usock.h" #include "src/util/output.h" #include "src/util/keyval_parse.h" #include "src/util/show_help.h" @@ -37,6 +36,7 @@ #include "src/mca/pif/base/base.h" #include "src/mca/pinstalldirs/base/base.h" #include "src/mca/psec/base/base.h" +#include "src/mca/ptl/base/base.h" #include "src/dstore/pmix_dstore.h" #include PMIX_EVENT_HEADER @@ -73,17 +73,7 @@ void pmix_rte_finalize(void) } /* cleanup communications */ - pmix_usock_finalize(); - - if (!pmix_globals.external_evbase) { - /* finalize the progress thread */ - (void)pmix_progress_thread_finalize(NULL); - } - - if (PMIX_PROC_SERVER != pmix_globals.proc_type && - 0 <= pmix_client_globals.myserver.sd) { - CLOSE_THE_SOCKET(pmix_client_globals.myserver.sd); - } + (void)pmix_mca_base_framework_close(&pmix_ptl_base_framework); #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) pmix_dstore_finalize(); #endif /* PMIX_ENABLE_DSTORE */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c index eb9c7ecd37..67306b560c 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c +++ b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c @@ -44,12 +44,13 @@ #include "src/mca/pif/base/base.h" #include "src/mca/pinstalldirs/base/base.h" #include "src/mca/psec/base/base.h" +#include "src/mca/ptl/base/base.h" #include "src/event/pmix_event.h" #include "src/include/types.h" -#include "src/usock/usock.h" #include "src/util/error.h" #include "src/util/keyval_parse.h" +#include "src/buffer_ops/buffer_ops.h" #include "src/runtime/pmix_rte.h" #include "src/runtime/pmix_progress_threads.h" @@ -82,11 +83,10 @@ PMIX_EXPORT pmix_globals_t pmix_globals = { int pmix_rte_init(pmix_proc_type_t type, pmix_info_t info[], size_t ninfo, - pmix_usock_cbfunc_t notifycbfunc) + pmix_ptl_cbfunc_t notifycbfunc) { int ret, debug_level; char *error = NULL, *evar; - char *param; size_t n; if( ++pmix_initialized != 1 ) { @@ -179,19 +179,21 @@ int pmix_rte_init(pmix_proc_type_t type, } pmix_bfrop_open(); -#if 0 - /* open the bfrops - we will select the active plugin later */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_bfrops_base_framework, 0)) ) { - error = "pmix_bfrops_base_open"; - goto return_error; - } - if( PMIX_SUCCESS != (ret = pmix_bfrop_base_select()) ) { - error = "pmix_bfrops_base_select"; - goto return_error; - } -#endif + /* the choice of modules to use when communicating with a peer + * will be done by the individual init functions and at the + * time of connection to that peer */ - /* open the psec and select the default module for this environment */ + /* open the ptl and select the active plugins */ + if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) { + error = "pmix_ptl_base_open"; + goto return_error; + } + if( PMIX_SUCCESS != (ret = pmix_ptl_base_select()) ) { + error = "pmix_ptl_base_select"; + goto return_error; + } + + /* open the psec and select the active plugins */ if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_psec_base_framework, 0))) { error = "pmix_psec_base_open"; goto return_error; @@ -200,13 +202,6 @@ int pmix_rte_init(pmix_proc_type_t type, error = "pmix_psec_base_select"; goto return_error; } - param = getenv("PMIX_SEC_MODULE"); // if directive was given, use it - pmix_globals.mypeer->compat.psec = pmix_psec_base_assign_module(param); - if (NULL == pmix_globals.mypeer->compat.psec) { - ret = PMIX_ERR_NOT_SUPPORTED; - error = "no psec module"; - goto return_error; - } /* initialize pif framework */ if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_pif_base_framework, 0))) { @@ -216,6 +211,8 @@ int pmix_rte_init(pmix_proc_type_t type, /* tell libevent that we need thread support */ pmix_event_use_threads(); + + /* if an external event base wasn't provide, create one */ if (!pmix_globals.external_evbase) { /* create an event base and progress thread for us */ if (NULL == (pmix_globals.evbase = pmix_progress_thread_init(NULL))) { @@ -225,33 +222,6 @@ int pmix_rte_init(pmix_proc_type_t type, } } -#if 0 - /* open the PMIx Transport Layer (ptl) - we will select the - * active plugin later */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) { - error = "pmix_ptl_base_open"; - goto return_error; - } - if( PMIX_SUCCESS != (ret = pmix_ptl_base_select()) ) { - error = "pmix_ptl_base_select"; - goto return_error; - } - - /* open the dstor framework - we will select the active - * plugin later */ - if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_pdstor_base_framework, 0)) ) { - error = "pmix_pdstor_base_open"; - goto return_error; - } - if( PMIX_SUCCESS != (ret = pmix_pdstor_base_select()) ) { - error = "pmix_pdstor_base_select"; - goto return_error; - } -#endif - - /* setup the usock service */ - pmix_usock_init(notifycbfunc); - return PMIX_SUCCESS; return_error: diff --git a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_rte.h b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_rte.h index 15665d6d8b..0ef36e271e 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_rte.h +++ b/opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_rte.h @@ -34,6 +34,7 @@ #include PMIX_EVENT_HEADER #include "src/include/pmix_globals.h" +#include "src/mca/ptl/ptl_types.h" BEGIN_C_DECLS @@ -58,7 +59,7 @@ extern const char pmix_version_string[]; */ pmix_status_t pmix_rte_init(pmix_proc_type_t type, pmix_info_t info[], size_t ninfo, - pmix_usock_cbfunc_t cbfunc); + pmix_ptl_cbfunc_t cbfunc); /** * Finalize the PMIX layer, including the MCA system. diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/Makefile.include b/opal/mca/pmix/pmix2x/pmix/src/server/Makefile.include index e3577865ae..7e7cacaba9 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/Makefile.include +++ b/opal/mca/pmix/pmix2x/pmix/src/server/Makefile.include @@ -20,5 +20,4 @@ sources += \ server/pmix_server.c \ server/pmix_server_ops.c \ server/pmix_server_regex.c \ - server/pmix_server_get.c \ - server/pmix_server_listener.c + server/pmix_server_get.c diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c index 3ee8a3610c..8209649e48 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c @@ -61,7 +61,7 @@ #include "src/mca/pinstalldirs/base/base.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/base/base.h" #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) #include "src/dstore/pmix_dstore.h" #endif /* PMIX_ENABLE_DSTORE */ @@ -74,60 +74,19 @@ pmix_server_globals_t pmix_server_globals = {{{0}}}; // local variables static char *security_mode = NULL; +static char *ptl_mode = NULL; static pid_t mypid; -static char *mytmpdir = NULL; -static char *systmpdir = NULL; // local functions for connection support -static void server_message_handler(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void server_message_handler(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata); + static inline int _my_client(const char *nspace, pmix_rank_t rank); -/* queue a message to be sent to one of our procs - must - * provide the following params: - * - * p - the peer object of the process - * t - tag to be sent to - * b - buffer to be sent - */ -void pmix_server_queue_message(int fd, short args, void *cbdata) -{ - pmix_usock_queue_t *queue = (pmix_usock_queue_t*)cbdata; - pmix_usock_send_t *snd; - pmix_output_verbose(2, pmix_globals.debug_output, - "[%s:%d] queue callback called: reply to %s:%d on tag %d", - __FILE__, __LINE__, - (queue->peer)->info->nptr->nspace, - (queue->peer)->info->rank, (queue->tag)); - snd = PMIX_NEW(pmix_usock_send_t); - snd->hdr.pindex = pmix_globals.pindex; - snd->hdr.tag = (queue->tag); - snd->hdr.nbytes = (queue->buf)->bytes_used; - snd->data = (queue->buf); - /* always start with the header */ - snd->sdptr = (char*)&snd->hdr; - snd->sdbytes = sizeof(pmix_usock_hdr_t); - - /* if there is no message on-deck, put this one there */ - if (NULL == (queue->peer)->send_msg) { - (queue->peer)->send_msg = snd; - } else { - /* add it to the queue */ - pmix_list_append(&(queue->peer)->send_queue, &snd->super); - } - /* ensure the send event is active */ - if (!(queue->peer)->send_ev_active) { - event_add(&(queue->peer)->send_event, 0); - (queue->peer)->send_ev_active = true; - } - PMIX_RELEASE(queue); -} - static pmix_status_t initialize_server_base(pmix_server_module_t *module) { - char *tdir, *evar; - char * pmix_pid; - pmix_listener_t *listener; + char *evar; /* look for our namespace, if one was given */ if (NULL == (evar = getenv("PMIX_SERVER_NAMESPACE"))) { @@ -154,7 +113,6 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module) PMIX_CONSTRUCT(&pmix_server_globals.events, pmix_list_t); PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t); PMIX_CONSTRUCT(&pmix_server_globals.notifications, pmix_ring_buffer_t); - PMIX_CONSTRUCT(&pmix_server_globals.listeners, pmix_list_t); pmix_ring_buffer_init(&pmix_server_globals.notifications, 256); pmix_output_verbose(2, pmix_globals.debug_output, @@ -164,65 +122,25 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module) memset(&pmix_host_server, 0, sizeof(pmix_server_module_t)); pmix_host_server = *module; - /* find the temp dir */ - if (NULL != mytmpdir) { - tdir = mytmpdir; - } else if (NULL == (tdir = getenv("TMPDIR"))) { - if (NULL == (tdir = getenv("TEMP"))) { - if (NULL == (tdir = getenv("TMP"))) { - tdir = "/tmp"; - } - } - } - - /* for now, just setup the v1.1 series rendezvous point - * we use the pid to reduce collisions */ - if (0 > asprintf(&pmix_pid, "%s/pmix-%d", tdir, mypid)) { - return PMIX_ERR_NOMEM; - } - - if ((strlen(pmix_pid) + 1) > sizeof(listener->address.sun_path)-1) { - pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true, tdir, pmix_pid); - free(pmix_pid); - return PMIX_ERR_INVALID_LENGTH; - } - - listener = PMIX_NEW(pmix_listener_t); - snprintf(listener->address.sun_path, sizeof(listener->address.sun_path)-1, "%s", pmix_pid); - if (0 > asprintf(&listener->uri, "%s:%lu:%s", pmix_globals.myid.nspace, - (unsigned long)pmix_globals.myid.rank, listener->address.sun_path)) { - free(pmix_pid); - return PMIX_ERR_NOMEM; - } - listener->varname = strdup("PMIX_SERVER_URI"); - listener->protocol = PMIX_PROTOCOL_V1; - pmix_list_append(&pmix_server_globals.listeners, &listener->super); - free(pmix_pid); - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:server constructed uri %s", listener->uri); - return PMIX_SUCCESS; } PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, pmix_info_t info[], size_t ninfo) { - pmix_usock_posted_recv_t *req; + pmix_ptl_posted_recv_t *req; pmix_status_t rc; size_t n, m; pmix_kval_t kv; - pmix_listener_t *lt; - bool need_listener; - int myhostnamelen = 30; - char myhostname[myhostnamelen]; - char *pmix_pid, *tdir; - char **protected = NULL; bool protect; - bool tool_support = false; - bool system_tool = false; - bool session_tool = false; - pmix_listener_t *tl; + char *protected[] = { + PMIX_USERID, + PMIX_GRPID, + PMIX_SOCKET_MODE, + PMIX_SERVER_TOOL_SUPPORT, + PMIX_SERVER_SYSTEM_SUPPORT, + NULL + }; if (0 < pmix_globals.init_cntr) { return PMIX_SUCCESS; @@ -238,26 +156,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, return rc; } - /* Check for the info keys that are not independent from - * initialize_server_base() and even may be needed there */ - if (NULL != info) { - for (n=0; n < ninfo; n++) { - if (0 == strcmp(info[n].key, PMIX_SERVER_TMPDIR) && - NULL == mytmpdir) { - mytmpdir = strdup(info[n].value.data.string); - /* push this onto our protected list of keys not - * to be passed to the clients */ - pmix_argv_append_nosize(&protected, PMIX_SERVER_TMPDIR); - } else if (0 == strcmp(info[n].key, PMIX_SYSTEM_TMPDIR) && - NULL == systmpdir) { - systmpdir = strdup(info[n].value.data.string); - /* push this onto our protected list of keys not - * to be passed to the clients */ - pmix_argv_append_nosize(&protected, PMIX_SYSTEM_TMPDIR); - } - } - } - if (0 != (rc = initialize_server_base(module))) { return rc; } @@ -268,149 +166,17 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, } #endif /* PMIX_ENABLE_DSTORE */ - /* check the info keys for a directive about the uid/gid - * to be set for the rendezvous file */ - if (NULL != info) { - for (n=0; n < ninfo; n++) { - if (0 == strcmp(info[n].key, PMIX_USERID)) { - /* the userid is in the uint32_t storage */ - PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { - lt->owner = info[n].value.data.uint32; - lt->owner_given = true; - } - /* push this onto our protected list of keys not - * to be passed to the clients */ - pmix_argv_append_nosize(&protected, PMIX_USERID); - } else if (0 == strcmp(info[n].key, PMIX_GRPID)) { - /* the grpid is in the uint32_t storage */ - PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { - lt->group = info[n].value.data.uint32; - lt->group_given = true; - } - /* push this onto our protected list of keys not - * to be passed to the clients */ - pmix_argv_append_nosize(&protected, PMIX_GRPID); - } else if (0 == strcmp(info[n].key, PMIX_SOCKET_MODE)) { - /* socket mode is in the uint32_t storage */ - PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { - lt->mode = info[n].value.data.uint32; - } - } else if (0 == strcmp(info[n].key, PMIX_SERVER_TOOL_SUPPORT)) { - /* defer processing to ensure we pickup any tmpdir - * directives before setting location */ - session_tool = true; - tool_support = true; - /* push this onto our protected list of keys not - * to be passed to the clients */ - pmix_argv_append_nosize(&protected, PMIX_SERVER_TOOL_SUPPORT); - } else if (0 == strcmp(info[n].key, PMIX_SERVER_SYSTEM_SUPPORT)) { - /* we are also the system tool server */ - system_tool = true; - tool_support = true; - /* push this onto our protected list of keys not - * to be passed to the clients */ - pmix_argv_append_nosize(&protected, PMIX_SERVER_TOOL_SUPPORT); - } - } - } - if (tool_support) { - /* Get up to 30 chars of hostname.*/ - gethostname(myhostname, myhostnamelen); - /* ensure it is NULL terminated */ - myhostname[myhostnamelen-1] = '\0'; - /* if we are to be the system tool, then we look for - * the system tmpdir and do not include a pid in - * the rendezvous point */ - if (system_tool) { - if (NULL != systmpdir) { - tdir = systmpdir; - } else if (NULL == (tdir = getenv("TMPDIR"))) { - if (NULL == (tdir = getenv("TEMP"))) { - if (NULL == (tdir = getenv("TMP"))) { - tdir = "/tmp"; - } - } - } - if (0 > asprintf(&pmix_pid, "%s/pmix.sys.%s", tdir, myhostname)) { - return PMIX_ERR_NOMEM; - } - if ((strlen(pmix_pid) + 1) > sizeof(tl->address.sun_path)-1) { - pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true, tdir, pmix_pid); - free(pmix_pid); - return PMIX_ERR_INVALID_LENGTH; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "%s:%d dropping system tool at %s", - pmix_globals.myid.nspace, - pmix_globals.myid.rank, pmix_pid); - /* create the listener for this point */ - tl = PMIX_NEW(pmix_listener_t); - tl -> address.sun_family = AF_UNIX; - tl->protocol = PMIX_PROTOCOL_TOOL; - snprintf(tl->address.sun_path, sizeof(tl->address.sun_path) - 1, "%s", pmix_pid); - free(pmix_pid); - pmix_list_append(&pmix_server_globals.listeners, &tl->super); - } - /* if we are a session tool, then we use the session tmpdir - * and append our pid */ - if (session_tool) { - if (NULL != mytmpdir) { - tdir = mytmpdir; - } else if (NULL == (tdir = getenv("TMPDIR"))) { - if (NULL == (tdir = getenv("TEMP"))) { - if (NULL == (tdir = getenv("TMP"))) { - tdir = "/tmp"; - } - } - } - /* mark this with my pid */ - if (0 > asprintf(&pmix_pid, "%s/pmix.%s.tool.%d", tdir, myhostname, mypid)) { - return PMIX_ERR_NOMEM; - } - if ((strlen(pmix_pid) + 1) > sizeof(tl->address.sun_path)-1) { - pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true, tdir, pmix_pid); - free(pmix_pid); - return PMIX_ERR_INVALID_LENGTH; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "%s:%d dropping session tool at %s", - pmix_globals.myid.nspace, - pmix_globals.myid.rank, pmix_pid); - /* create the listener for this point */ - tl = PMIX_NEW(pmix_listener_t); - tl -> address.sun_family = AF_UNIX; - tl->protocol = PMIX_PROTOCOL_TOOL; - snprintf(tl->address.sun_path, sizeof(tl->address.sun_path) - 1, "%s", pmix_pid); - free(pmix_pid); - pmix_list_append(&pmix_server_globals.listeners, &tl->super); - } - /* we don't provide a URI for this listener as we don't pass - * the TOOL connection URI to a child process */ - pmix_server_globals.tool_connections_allowed = true; - } - /* setup the wildcard recv for inbound messages from clients */ - req = PMIX_NEW(pmix_usock_posted_recv_t); + req = PMIX_NEW(pmix_ptl_posted_recv_t); req->tag = UINT32_MAX; req->cbfunc = server_message_handler; /* add it to the end of the list of recvs */ - pmix_list_append(&pmix_usock_globals.posted_recvs, &req->super); + pmix_list_append(&pmix_ptl_globals.posted_recvs, &req->super); - /* start listening */ - need_listener = false; - PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { - if (PMIX_SUCCESS != pmix_prepare_listening(lt, &need_listener)) { - pmix_show_help("help-pmix-server.txt", "listener-failed-start", true, lt->address.sun_path); - PMIx_server_finalize(); - return PMIX_ERR_INIT; - } - } - if (need_listener) { - if (PMIX_SUCCESS != pmix_start_listening()) { - pmix_show_help("help-pmix-server.txt", "listener-thread-start", true); - PMIx_server_finalize(); - return PMIX_ERR_INIT; - } + if (PMIX_SUCCESS != pmix_ptl_base_start_listening(info, ninfo)) { + pmix_show_help("help-pmix-server.txt", "listener-thread-start", true); + PMIx_server_finalize(); + return PMIX_ERR_INIT; } /* check the info keys for info we @@ -420,12 +186,10 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, for (n=0; n < ninfo; n++) { /* check the list of protected keys */ protect = false; - if (NULL != protected) { - for (m=0; NULL != protected[m]; m++) { - if (0 == strcmp(info[n].key, protected[m])) { - protect = true; - break; - } + for (m=0; NULL != protected[m]; m++) { + if (0 == strcmp(info[n].key, protected[m])) { + protect = true; + break; } } if (protect) { @@ -451,6 +215,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, PMIX_DESTRUCT(&kv); } + /* get our available security modules */ + security_mode = pmix_psec.get_available_modules(); + + /* get our available ptl modules */ + ptl_mode = pmix_ptl.get_available_modules(); + ++pmix_globals.init_cntr; return PMIX_SUCCESS; @@ -471,14 +241,13 @@ static void cleanup_server_state(void) PMIX_LIST_DESTRUCT(&pmix_server_globals.remote_pnd); PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs); PMIX_DESTRUCT(&pmix_server_globals.gdata); - PMIX_LIST_DESTRUCT(&pmix_server_globals.listeners); - if (NULL != mytmpdir) { - free(mytmpdir); + if (NULL != security_mode) { + free(security_mode); } - if (NULL != systmpdir) { - free(systmpdir); + if (NULL != ptl_mode) { + free(ptl_mode); } pmix_bfrop_close(); @@ -496,9 +265,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) pmix_output_verbose(2, pmix_globals.debug_output, "pmix:server finalize called"); - if (pmix_server_globals.listen_thread_active) { - pmix_stop_listening(); - } + pmix_ptl_base_stop_listening(); cleanup_server_state(); pmix_output_verbose(2, pmix_globals.debug_output, @@ -1081,13 +848,15 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char * (void)snprintf(rankstr, 127, "%d", proc->rank); pmix_setenv("PMIX_RANK", rankstr, true, env); /* pass our rendezvous info */ - PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { + PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) { if (NULL != lt->uri && NULL != lt->varname) { pmix_setenv(lt->varname, lt->uri, true, env); } } - /* pass our active security mode */ + /* pass our active security modules */ pmix_setenv("PMIX_SECURITY_MODE", security_mode, true, env); + /* pass our available ptl modules */ + pmix_setenv("PMIX_PTL_MODULE", ptl_mode, true, env); #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) /* pass dstore path to files */ @@ -1661,12 +1430,17 @@ static void op_cbfunc(pmix_status_t status, void *cbdata) if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(reply, &status, 1, PMIX_STATUS))) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(reply); + PMIX_RELEASE(cd); return; } /* the function that created the server_caddy did a * retain on the peer, so we don't have to worry about * it still being present - send a copy to the originator */ - PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply); + if (PMIX_SUCCESS != (rc = pmix_ptl.send_oneway(cd->peer, reply, cd->hdr.tag))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(reply); + } + /* cleanup */ PMIX_RELEASE(cd); } @@ -2237,7 +2011,7 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, pmix_bfrop.copy_payload(reply, &(pmix_server_globals.gdata)); #endif PMIX_SERVER_QUEUE_REPLY(peer, tag, reply); - return PMIX_SUCCESS; + return PMIX_SUCCESS; // don't reply twice } if (PMIX_ABORT_CMD == cmd) { @@ -2250,7 +2024,10 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, if (PMIX_COMMIT_CMD == cmd) { rc = pmix_server_commit(peer, buf); - return rc; + reply = PMIX_NEW(pmix_buffer_t); + pmix_bfrop.pack(reply, &rc, 1, PMIX_STATUS); + PMIX_SERVER_QUEUE_REPLY(peer, tag, reply); + return PMIX_SUCCESS; // don't reply twice } if (PMIX_FENCENB_CMD == cmd) { @@ -2390,7 +2167,8 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag, return PMIX_ERR_NOT_SUPPORTED; } -static void server_message_handler(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr, +static void server_message_handler(struct pmix_peer_t *pr, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_peer_t *peer = (pmix_peer_t*)pr; diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c index a98b7effd4..0a2f2f894a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c @@ -52,7 +52,6 @@ #include "src/util/error.h" #include "src/util/output.h" #include "src/util/pmix_environ.h" -#include "src/usock/usock.h" #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) #include "src/dstore/pmix_dstore.h" #endif /* PMIX_ENABLE_DSTORE */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_listener.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_listener.c deleted file mode 100644 index 871d0ecc96..0000000000 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_listener.c +++ /dev/null @@ -1,972 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2014-2015 Artem Y. Polyakov . - * All rights reserved. - * Copyright (c) 2016 Mellanox Technologies, Inc. - * All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include - -#include -#include -#include - -#include -#include "src/include/pmix_globals.h" - -#ifdef HAVE_STRING_H -#include -#endif -#include -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_SYS_UN_H -#include -#endif -#ifdef HAVE_SYS_UIO_H -#include -#endif -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#include -#include -#include PMIX_EVENT_HEADER -#include - -#include "src/class/pmix_list.h" -#include "src/buffer_ops/buffer_ops.h" -#include "src/util/argv.h" -#include "src/util/error.h" -#include "src/util/fd.h" -#include "src/util/getid.h" -#include "src/util/output.h" -#include "src/util/pmix_environ.h" -#include "src/util/strnlen.h" -#include "src/usock/usock.h" -#include "src/mca/psec/psec.h" - -#include "pmix_server_ops.h" - -// local functions for connection support -static void* listen_thread(void *obj); -static void listener_cb(int incoming_sd, void *cbdata); -static void connection_handler(int incoming_sd, short flags, void* cbdata); -static void tool_handler(int incoming_sd, short flags, void* cbdata); -static char *myversion = NULL; -static pthread_t engine; - -/* - * start listening on our rendezvous file - */ -pmix_status_t pmix_prepare_listening(pmix_listener_t *lt, bool *need_listener) -{ - int flags; - pmix_status_t rc; - socklen_t addrlen; - char *ptr; - struct sockaddr_un *address = <->address; - - /* create a listen socket for incoming connection attempts */ - lt->socket = socket(PF_UNIX, SOCK_STREAM, 0); - if (lt->socket < 0) { - printf("%s:%d socket() failed\n", __FILE__, __LINE__); - return PMIX_ERROR; - } - /* Set the socket to close-on-exec so that no children inherit - * this FD */ - if (pmix_fd_set_cloexec(lt->socket) != PMIX_SUCCESS) { - CLOSE_THE_SOCKET(lt->socket); - return PMIX_ERROR; - } - - - addrlen = sizeof(struct sockaddr_un); - if (bind(lt->socket, (struct sockaddr*)address, addrlen) < 0) { - printf("%s:%d bind() failed\n", __FILE__, __LINE__); - return PMIX_ERROR; - } - /* chown as required */ - if (lt->owner_given) { - if (0 != chown(address->sun_path, lt->owner, -1)) { - pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno)); - goto sockerror; - } - } - if (lt->group_given) { - if (0 != chown(address->sun_path, -1, lt->group)) { - pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno)); - goto sockerror; - } - } - /* set the mode as required */ - if (0 != chmod(address->sun_path, lt->mode)) { - pmix_output(0, "CANNOT CHMOD socket %s: %s", address->sun_path, strerror (errno)); - goto sockerror; - } - - /* setup listen backlog to maximum allowed by kernel */ - if (listen(lt->socket, SOMAXCONN) < 0) { - printf("%s:%d listen() failed\n", __FILE__, __LINE__); - goto sockerror; - } - - /* set socket up to be non-blocking, otherwise accept could block */ - if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) { - printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__); - goto sockerror; - } - flags |= O_NONBLOCK; - if (fcntl(lt->socket, F_SETFL, flags) < 0) { - printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__); - goto sockerror; - } - - if (NULL == myversion) { - /* setup my version for validating connections - we - * only check the major version numbers */ - myversion = strdup(PMIX_VERSION); - /* find the first '.' */ - ptr = strchr(myversion, '.'); - if (NULL != ptr) { - ++ptr; - /* stop it at the second '.', if present */ - if (NULL != (ptr = strchr(ptr, '.'))) { - *ptr = '\0'; - } - } - } - - /* if the server will listen for us, then ask it to do so now */ - rc = PMIX_ERR_NOT_SUPPORTED; - if (NULL != pmix_host_server.listener) { - rc = pmix_host_server.listener(lt->socket, listener_cb, (void*)lt); - } - - if (PMIX_SUCCESS != rc && !pmix_server_globals.listen_thread_active) { - *need_listener = true; - } - - return PMIX_SUCCESS; - -sockerror: - (void)close(lt->socket); - lt->socket = -1; - return PMIX_ERROR; -} - -pmix_status_t pmix_start_listening(void) { - /*** spawn internal listener thread */ - if (0 > pipe(pmix_server_globals.stop_thread)) { - PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO); - return PMIX_ERR_OUT_OF_RESOURCE; - } - /* Make sure the pipe FDs are set to close-on-exec so that - they don't leak into children */ - if (pmix_fd_set_cloexec(pmix_server_globals.stop_thread[0]) != PMIX_SUCCESS || - pmix_fd_set_cloexec(pmix_server_globals.stop_thread[1]) != PMIX_SUCCESS) { - PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO); - close(pmix_server_globals.stop_thread[0]); - close(pmix_server_globals.stop_thread[1]); - return PMIX_ERR_OUT_OF_RESOURCE; - } - /* fork off the listener thread */ - pmix_server_globals.listen_thread_active = true; - if (0 > pthread_create(&engine, NULL, listen_thread, NULL)) { - pmix_server_globals.listen_thread_active = false; - return PMIX_ERROR; - } - - return PMIX_SUCCESS; -} - -void pmix_stop_listening(void) -{ - int i; - pmix_listener_t *lt; - - pmix_output_verbose(8, pmix_globals.debug_output, - "listen_thread: shutdown"); - - if (!pmix_server_globals.listen_thread_active) { - /* nothing we can do */ - return; - } - - /* mark it as inactive */ - pmix_server_globals.listen_thread_active = false; - /* use the block to break it loose just in - * case the thread is blocked in a call to select for - * a long time */ - i=1; - if (0 > write(pmix_server_globals.stop_thread[1], &i, sizeof(int))) { - return; - } - /* wait for thread to exit */ - pthread_join(engine, NULL); - /* close the sockets to remove the connection points */ - PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { - CLOSE_THE_SOCKET(lt->socket); - lt->socket = -1; - } -} - -static void* listen_thread(void *obj) -{ - int rc, max, accepted_connections; - socklen_t addrlen = sizeof(struct sockaddr_storage); - pmix_pending_connection_t *pending_connection; - struct timeval timeout; - fd_set readfds; - pmix_listener_t *lt; - - pmix_output_verbose(8, pmix_globals.debug_output, - "listen_thread: active"); - - - while (pmix_server_globals.listen_thread_active) { - FD_ZERO(&readfds); - max = -1; - PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { - FD_SET(lt->socket, &readfds); - max = (lt->socket > max) ? lt->socket : max; - } - /* add the stop_thread fd */ - FD_SET(pmix_server_globals.stop_thread[0], &readfds); - max = (pmix_server_globals.stop_thread[0] > max) ? pmix_server_globals.stop_thread[0] : max; - - /* set timeout interval */ - timeout.tv_sec = 2; - timeout.tv_usec = 0; - - /* Block in a select to avoid hammering the cpu. If a connection - * comes in, we'll get woken up right away. - */ - rc = select(max + 1, &readfds, NULL, NULL, &timeout); - if (!pmix_server_globals.listen_thread_active) { - /* we've been asked to terminate */ - close(pmix_server_globals.stop_thread[0]); - close(pmix_server_globals.stop_thread[1]); - return NULL; - } - if (rc < 0) { - continue; - } - - /* Spin accepting connections until all active listen sockets - * do not have any incoming connections, pushing each connection - * onto the event queue for processing - */ - do { - accepted_connections = 0; - PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) { - - /* according to the man pages, select replaces the given descriptor - * set with a subset consisting of those descriptors that are ready - * for the specified operation - in this case, a read. So we need to - * first check to see if this file descriptor is included in the - * returned subset - */ - if (0 == FD_ISSET(lt->socket, &readfds)) { - /* this descriptor is not included */ - continue; - } - - /* this descriptor is ready to be read, which means a connection - * request has been received - so harvest it. All we want to do - * here is accept the connection and push the info onto the event - * library for subsequent processing - we don't want to actually - * process the connection here as it takes too long, and so the - * OS might start rejecting connections due to timeout. - */ - pending_connection = PMIX_NEW(pmix_pending_connection_t); - pending_connection->protocol = lt->protocol; - if (PMIX_PROTOCOL_TOOL == lt->protocol) { - event_assign(&pending_connection->ev, pmix_globals.evbase, -1, - EV_WRITE, tool_handler, pending_connection); - } else { - event_assign(&pending_connection->ev, pmix_globals.evbase, -1, - EV_WRITE, connection_handler, pending_connection); - } - pending_connection->sd = accept(lt->socket, - (struct sockaddr*)&(pending_connection->addr), - &addrlen); - if (pending_connection->sd < 0) { - PMIX_RELEASE(pending_connection); - if (pmix_socket_errno != EAGAIN || - pmix_socket_errno != EWOULDBLOCK) { - if (EMFILE == pmix_socket_errno || - ENOBUFS == pmix_socket_errno || - ENOMEM == pmix_socket_errno) { - PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - } else if (EINVAL == pmix_socket_errno || - EINTR == pmix_socket_errno) { - /* race condition at finalize */ - goto done; - } else if (ECONNABORTED == pmix_socket_errno) { - /* they aborted the attempt */ - continue; - } else { - pmix_output(0, "listen_thread: accept() failed: %s (%d).", - strerror(pmix_socket_errno), pmix_socket_errno); - } - goto done; - } - continue; - } - - pmix_output_verbose(8, pmix_globals.debug_output, - "listen_thread: new connection: (%d, %d)", - pending_connection->sd, pmix_socket_errno); - /* activate the event */ - event_active(&pending_connection->ev, EV_WRITE, 1); - accepted_connections++; - } - } while (accepted_connections > 0); - } - - done: - pmix_server_globals.listen_thread_active = false; - return NULL; -} - -static void listener_cb(int incoming_sd, void *cbdata) -{ - pmix_pending_connection_t *pending_connection; - pmix_listener_t *lt = (pmix_listener_t*)cbdata; - - /* throw it into our event library for processing */ - pmix_output_verbose(8, pmix_globals.debug_output, - "listen_cb: pushing new connection %d into evbase", - incoming_sd); - pending_connection = PMIX_NEW(pmix_pending_connection_t); - pending_connection->sd = incoming_sd; - pending_connection->protocol = lt->protocol; - event_assign(&pending_connection->ev, pmix_globals.evbase, -1, - EV_WRITE, connection_handler, pending_connection); - event_active(&pending_connection->ev, EV_WRITE, 1); -} - -/* process the callback with tool connection info */ -static void process_cbfunc(int sd, short args, void *cbdata) -{ - pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; - pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cd->cbdata; - pmix_nspace_t *nptr; - pmix_rank_info_t *info; - int rc; - - /* send this status as well so they don't hang */ - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&cd->status, sizeof(pmix_status_t)))) { - PMIX_ERROR_LOG(rc); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - - /* if the request failed, then we are done */ - if (PMIX_SUCCESS != cd->status) { - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - - /* send the nspace back to the tool */ - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) { - PMIX_ERROR_LOG(rc); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - - /* send my nspace back to the tool */ - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1))) { - PMIX_ERROR_LOG(rc); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - - /* send my rank back to the tool */ - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&pmix_globals.myid.rank, sizeof(int)))) { - PMIX_ERROR_LOG(rc); - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - return; - } - - /* add this nspace to our pool */ - nptr = PMIX_NEW(pmix_nspace_t); - (void)strncpy(nptr->nspace, cd->proc.nspace, PMIX_MAX_NSLEN); - nptr->server = PMIX_NEW(pmix_server_nspace_t); - pmix_list_append(&pmix_globals.nspaces, &nptr->super); - /* add this tool rank to the nspace */ - info = PMIX_NEW(pmix_rank_info_t); - PMIX_RETAIN(nptr); - info->nptr = nptr; - info->rank = 0; - /* need to include the uid/gid for validation */ - info->uid = pnd->uid; - info->gid = pnd->gid; - pmix_list_append(&nptr->server->ranks, &info->super); - - /* setup a peer object for this tool */ - pmix_peer_t *peer = PMIX_NEW(pmix_peer_t); - PMIX_RETAIN(info); - peer->info = info; - peer->proc_cnt = 1; - peer->sd = pnd -> sd; - - /* get the appropriate compatibility modules */ - // peer->comm.type = pnd->buffer_type; - // peer->comm.bfrops = pmix_bfrops_base_assign_module(pnd->bfrop); - peer->compat.psec = pmix_psec_base_assign_module(pnd->psec); - - /* if a credential is available, then check it */ - if (NULL != peer->compat.psec->validate_cred) { - if (PMIX_SUCCESS != (rc = peer->compat.psec->validate_cred(peer, pnd->cred))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "validation of tool credential failed: %s", - PMIx_Error_string(rc)); - PMIX_RELEASE(peer); - pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); // will release the info object - CLOSE_THE_SOCKET(pnd->sd); - goto done; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "tool credential validated"); - /* send them success */ - rc = PMIX_SUCCESS; - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(peer); - pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); // will release the info object - CLOSE_THE_SOCKET(pnd->sd); - goto done; - } - } else if (NULL != peer->compat.psec->server_handshake) { - /* execute the handshake if the security mode calls for it */ - pmix_output_verbose(2, pmix_globals.debug_output, - "connect-ack executing handshake"); - rc = PMIX_ERR_READY_FOR_HANDSHAKE; - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(sd, (char*)&rc, sizeof(int)))) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(peer); - pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); // will release the info object - CLOSE_THE_SOCKET(pnd->sd); - goto done; - } - if (PMIX_SUCCESS != (rc = peer->compat.psec->server_handshake(peer))) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE(peer); - pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); // will release the info object - CLOSE_THE_SOCKET(pnd->sd); - goto done; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "tool connect-ack handshake complete"); - } else { - /* this is not allowed */ - PMIX_RELEASE(peer); - pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); // will release the info object - CLOSE_THE_SOCKET(pnd->sd); - goto done; - } - - /* set the socket non-blocking for all further operations */ - pmix_usock_set_nonblocking(pnd->sd); - - if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) { - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); - PMIX_RELEASE(peer); - pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super); - PMIX_RELEASE(nptr); // will release the info object - /* probably cannot send an error reply if we are out of memory */ - return; - } - - /* start the events for this tool */ - event_assign(&peer->recv_event, pmix_globals.evbase, pnd->sd, - EV_READ|EV_PERSIST, pmix_usock_recv_handler, peer); - event_add(&peer->recv_event, NULL); - peer->recv_ev_active = true; - event_assign(&peer->send_event, pmix_globals.evbase, pnd->sd, - EV_WRITE|EV_PERSIST, pmix_usock_send_handler, peer); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:server tool %s:%d has connected on socket %d", - peer->info->nptr->nspace, peer->info->rank, peer->sd); - - done: - PMIX_RELEASE(pnd); - PMIX_RELEASE(cd); -} - -/* receive a callback from the host RM with an nspace - * for a connecting tool */ -static void cnct_cbfunc(pmix_status_t status, - pmix_proc_t *proc, void *cbdata) -{ - pmix_setup_caddy_t *cd; - - /* need to thread-shift this into our context */ - cd = PMIX_NEW(pmix_setup_caddy_t); - cd->status = status; - (void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN); - cd->cbdata = cbdata; - PMIX_THREADSHIFT(cd, process_cbfunc); -} - -/* Parse init-ack message: - * NSPACE<0>VERSION<0>BFROP<0>SEC<0>BUFTYPE<0>[CRED<0>] - */ -static pmix_status_t parse_connect_ack (char *msg, - pmix_listener_protocol_t protocol, - int len, - char **nspace, pmix_rank_t *rank, - char **version, - char **bfrop, char **sec, - pmix_bfrop_buffer_type_t *type, - char **cred) -{ - int msglen; - - if (PMIX_PROTOCOL_TOOL != protocol) { - PMIX_STRNLEN(msglen, msg, len); - if (msglen < len) { - *nspace = msg; - msg += strlen(*nspace) + 1; - len -= strlen(*nspace) + 1; - } else { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - return PMIX_ERR_BAD_PARAM; - } - - if ((int)sizeof(pmix_rank_t) <= len) { - memcpy(rank, msg, sizeof(pmix_rank_t)); - msg += sizeof(pmix_rank_t); - len -= sizeof(pmix_rank_t); - } else { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - return PMIX_ERR_BAD_PARAM; - } - } - - PMIX_STRNLEN(msglen, msg, len); - if (msglen < len) { - *version = msg; - msg += strlen(*version) + 1; - len -= strlen(*version) + 1; - } else { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - return PMIX_ERR_BAD_PARAM; - } - -#if 0 - PMIX_STRNLEN(msglen, msg, len); - if (msglen < len) { - *bfrop = msg; - msg += strlen(*bfrop) + 1; - len -= strlen(*bfrop) + 1; - } else { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - return PMIX_ERR_BAD_PARAM; - } -#endif - - PMIX_STRNLEN(msglen, msg, len); - if (msglen < len) { - *sec = msg; - msg += strlen(*sec) + 1; - len -= strlen(*sec) + 1; - } else { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - return PMIX_ERR_BAD_PARAM; - } - -#if 0 - if ((int)sizeof(pmix_bfrop_buffer_type_t) <= len) { - memcpy(type, msg, sizeof(pmix_bfrop_buffer_type_t)); - msg += sizeof(pmix_bfrop_buffer_type_t); - len -= sizeof(pmix_bfrop_buffer_type_t); - } else { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - return PMIX_ERR_BAD_PARAM; - } -#endif - - PMIX_STRNLEN(msglen, msg, len); - if (msglen < len) - *cred = msg; - else { - *cred = NULL; - } - - return PMIX_SUCCESS; -} - -/* Receive the peer's identification info from a newly - * connected socket and verify the expected response. - */ -static pmix_status_t pmix_server_authenticate(pmix_pending_connection_t *pnd, - pmix_rank_t *out_rank, - pmix_peer_t **peer) -{ - char *msg, *nspace=NULL, *version=NULL, *bfrop=NULL, *sec=NULL, *cred=NULL; - pmix_status_t rc; - pmix_rank_t rank=PMIX_RANK_UNDEF; - pmix_usock_hdr_t hdr; - pmix_nspace_t *nptr, *tmp; - pmix_rank_info_t *info; - pmix_peer_t *psave = NULL; - bool found; - pmix_proc_t proc; - uid_t uid; - gid_t gid; - pmix_bfrop_buffer_type_t buffer_type = 0; - - pmix_output_verbose(2, pmix_globals.debug_output, - "RECV CONNECT ACK FROM PEER ON SOCKET %d", - pnd->sd); - - /* ensure all is zero'd */ - memset(&hdr, 0, sizeof(pmix_usock_hdr_t)); - if (NULL != peer) { - *peer = NULL; - } - - /* get the header */ - if (PMIX_SUCCESS != (rc = pmix_usock_recv_blocking(pnd->sd, (char*)&hdr, sizeof(pmix_usock_hdr_t)))) { - return rc; - } - - /* get the id, authentication and version payload (and possibly - * security credential) - to guard against potential attacks, - * we'll set an arbitrary limit per a define */ - if (PMIX_MAX_CRED_SIZE < hdr.nbytes) { - return PMIX_ERR_BAD_PARAM; - } - if (NULL == (msg = (char*)malloc(hdr.nbytes))) { - return PMIX_ERR_OUT_OF_RESOURCE; - } - if (PMIX_SUCCESS != pmix_usock_recv_blocking(pnd->sd, msg, hdr.nbytes)) { - /* unable to complete the recv */ - pmix_output_verbose(2, pmix_globals.debug_output, - "unable to complete recv of connect-ack with client ON SOCKET %d", - pnd->sd); - free(msg); - return PMIX_ERR_UNREACH; - } - if (PMIX_SUCCESS != (rc = parse_connect_ack(msg, pnd->protocol, - hdr.nbytes, &nspace, - &rank, &version, - &bfrop, &sec, - &buffer_type, &cred))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "error parsing connect-ack from client ON SOCKET %d", pnd->sd); - free(msg); - return rc; - } - - /* if the attaching process is a tool, then kickoff that procedure */ - if (PMIX_PROTOCOL_TOOL == pnd->protocol) { - /* get the tool socket's uid and gid so we can pass them to - * the host RM for validation */ - if (PMIX_SUCCESS != (rc = pmix_util_getid(pnd->sd, &uid, &gid))) { - free(msg); - return rc; - } - /* we pass this info in an array of pmix_info_t structs, - * so set that up here */ - pnd->ninfo = 2; - PMIX_INFO_CREATE(pnd->info, pnd->ninfo); - (void)strncpy(pnd->info[0].key, PMIX_USERID, PMIX_MAX_KEYLEN); - pnd->info[0].value.type = PMIX_UINT32; - pnd->info[0].value.data.uint32 = uid; - (void)strncpy(pnd->info[1].key, PMIX_GRPID, PMIX_MAX_KEYLEN); - pnd->info[0].value.type = PMIX_UINT32; - pnd->info[0].value.data.uint32 = gid; - /* pass along the bfrop, buffer_type, and sec fields so - * we can assign them once we create a peer object */ - pnd->uid = uid; - pnd->gid = gid; - if (NULL != bfrop) { - pnd->bfrop = strdup(bfrop); - } - if (NULL != sec) { - pnd->psec = strdup(sec); - } - pnd->buffer_type = buffer_type; - /* pass along the credential for later */ - if (NULL != cred) { - pnd->cred = strdup(cred); - } - /* release the msg */ - free(msg); - /* request an nspace for this requestor - it will - * automatically be assigned rank=0 */ - pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd); - return PMIX_ERR_OPERATION_IN_PROGRESS; - } - - /* if the attaching process is not a tool, then set it up as - * a known peer */ - pmix_output_verbose(2, pmix_globals.debug_output, - "connect-ack recvd from peer %s:%d:%s", - nspace, rank, version); - - /* do not check the version - we only retain it at this - * time in case we need to check it at some future date. - * For now, our intent is to retain backward compatibility - * and so we will assume that all versions are compatible. */ - - /* see if we know this nspace */ - nptr = NULL; - PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_nspace_t) { - if (0 == strcmp(tmp->nspace, nspace)) { - nptr = tmp; - break; - } - } - if (NULL == nptr) { - /* we don't know this namespace, reject it */ - free(msg); - /* send an error reply to the client */ - rc = PMIX_ERR_NOT_FOUND; - goto error; - } - - /* see if we have this peer in our list */ - info = NULL; - found = false; - PMIX_LIST_FOREACH(info, &nptr->server->ranks, pmix_rank_info_t) { - if (info->rank == rank) { - found = true; - break; - } - } - if (!found) { - /* rank unknown, reject it */ - free(msg); - /* send an error reply to the client */ - rc = PMIX_ERR_NOT_FOUND; - goto error; - } - *out_rank = rank; - - /* a peer can connect on multiple sockets since it can fork/exec - * a child that also calls PMIx_Init, so add it here if necessary. - * Create the tracker for this peer */ - psave = PMIX_NEW(pmix_peer_t); - if (NULL == psave) { - rc = PMIX_ERR_NOMEM; - goto error; - } - PMIX_RETAIN(info); - psave->info = info; - info->proc_cnt++; /* increase number of processes on this rank */ - psave->sd = pnd->sd; - if (0 > (psave->index = pmix_pointer_array_add(&pmix_server_globals.clients, psave))) { - free(msg); - PMIX_RELEASE(psave); - /* probably cannot send an error reply if we are out of memory */ - return PMIX_ERR_OUT_OF_RESOURCE; - } - if (PMIX_PROTOCOL_V1 == pnd->protocol) { - /* we ignore the version string from 1.1 and before */ - /* assign the 1.1 bfrops module to this peer */ - // psave->compat.type = buffer_type; - // psave->compat.bfrops = pmix_bfrops_base_assign_module("1.1"); - /* take the default sec module as that is what we - * told them to use */ - psave->compat.psec = pmix_psec_base_assign_module(NULL); - } else if (PMIX_PROTOCOL_V2 == pnd->protocol) { - /* the newer protocol contains two fields: - * (a) a string containing the supported bfrops modules - * (b) a string containing the supported sec modules - */ - /* set the bfrops module to match this peer */ - // psave->compat.type = buffer_type; - // psave->compat.bfrops = pmix_bfrops_base_assign_module(bfrop); - /* set the sec module to match this peer */ - psave->compat.psec = pmix_psec_base_assign_module(sec); - } else { - /* unrecognized */ - free(msg); - pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); - PMIX_RELEASE(psave); - PMIX_RELEASE(info); - return PMIX_ERR_BAD_PARAM; - } - free(msg); - - /* see if there is a credential */ - if (NULL != psave->compat.psec->validate_cred) { - if (PMIX_SUCCESS != (rc = psave->compat.psec->validate_cred(psave, cred))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "validation of client credential failed"); - pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); - PMIX_RELEASE(psave); - /* send an error reply to the client */ - goto error; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "client credential validated"); - /* send them success */ - rc = PMIX_SUCCESS; - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) { - PMIX_ERROR_LOG(rc); - pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); - PMIX_RELEASE(psave); - return rc; - } - } else if (NULL != pmix_globals.mypeer->compat.psec->server_handshake) { - pmix_output_verbose(2, pmix_globals.debug_output, - "connect-ack executing handshake"); - rc = PMIX_ERR_READY_FOR_HANDSHAKE; - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) { - PMIX_ERROR_LOG(rc); - pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); - PMIX_RELEASE(psave); - return rc; - } - if (PMIX_SUCCESS != (rc = pmix_globals.mypeer->compat.psec->server_handshake(psave))) { - PMIX_ERROR_LOG(rc); - pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); - PMIX_RELEASE(psave); - return rc; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "connect-ack handshake complete"); - } else { - /* this is not allowed */ - pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); - PMIX_RELEASE(psave); - /* send an error reply to the client */ - rc = PMIX_ERR_INVALID_CRED; - goto error; - } - - /* send the client's array index */ - if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) { - PMIX_ERROR_LOG(rc); - pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); - PMIX_RELEASE(psave); - return rc; - } - - pmix_output_verbose(2, pmix_globals.debug_output, - "connect-ack from client completed"); - - *peer = psave; - /* let the host server know that this client has connected */ - if (NULL != pmix_host_server.client_connected) { - (void)strncpy(proc.nspace, psave->info->nptr->nspace, PMIX_MAX_NSLEN); - proc.rank = psave->info->rank; - rc = pmix_host_server.client_connected(&proc, psave->info->server_object, - NULL, NULL); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - } - } - return rc; - - error: - /* send an error reply to the client */ - if (PMIX_SUCCESS != pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int))) { - PMIX_ERROR_LOG(rc); - } - return rc; -} - -/* - * Handler for accepting client connections from the event library - */ -static void connection_handler(int sd, short flags, void* cbdata) -{ - pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata; - pmix_peer_t *peer; - pmix_rank_t rank; - pmix_status_t status; - pmix_output_verbose(8, pmix_globals.debug_output, - "connection_handler: new connection: %d", - pnd->sd); - - /* ensure the socket is in blocking mode */ - pmix_usock_set_blocking(pnd->sd); - - /* - * Receive identifier info from the client and authenticate it - the - * function will lookup and return the peer object if the connection - * is successfully authenticated */ - if (PMIX_SUCCESS != (status = pmix_server_authenticate(pnd, &rank, &peer))) { - if (PMIX_ERR_OPERATION_IN_PROGRESS != status) { - CLOSE_THE_SOCKET(pnd->sd); - } - return; - } - - pmix_usock_set_nonblocking(pnd->sd); - - /* start the events for this client */ - event_assign(&peer->recv_event, pmix_globals.evbase, pnd->sd, - EV_READ|EV_PERSIST, pmix_usock_recv_handler, peer); - event_add(&peer->recv_event, NULL); - peer->recv_ev_active = true; - event_assign(&peer->send_event, pmix_globals.evbase, pnd->sd, - EV_WRITE|EV_PERSIST, pmix_usock_send_handler, peer); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix:server client %s:%u has connected on socket %d", - peer->info->nptr->nspace, peer->info->rank, peer->sd); - PMIX_RELEASE(pnd); -} - -/* - * Handler for accepting tool connections from the event library - */ -static void tool_handler(int sd, short flags, void* cbdata) -{ - pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata; - - pmix_output_verbose(1, pmix_globals.debug_output, - "tool_handler: new tool connection: %d", - pnd->sd); - - /* if the server doesn't support this, then abort now */ - if (NULL == pmix_host_server.tool_connected) { - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - } - - /* ensure the socket is in blocking mode */ - pmix_usock_set_blocking(pnd->sd); - - /* initiate the authentication handshake */ - if (PMIX_ERR_OPERATION_IN_PROGRESS != pmix_server_authenticate(pnd, NULL, NULL)) { - CLOSE_THE_SOCKET(pnd->sd); - PMIX_RELEASE(pnd); - } -} diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c index 8bb7d704b8..e172f0238b 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c @@ -54,7 +54,6 @@ #include "src/util/error.h" #include "src/util/output.h" #include "src/util/pmix_environ.h" -#include "src/usock/usock.h" #include "pmix_server_ops.h" @@ -1594,34 +1593,6 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_local_t, pmix_list_item_t, lmcon, lmdes); -static void pccon(pmix_pending_connection_t *p) -{ - memset(p->nspace, 0, PMIX_MAX_NSLEN+1); - p->info = NULL; - p->ninfo = 0; - p->bfrop = NULL; - p->psec = NULL; - p->cred = NULL; -} -static void pcdes(pmix_pending_connection_t *p) -{ - if (NULL != p->info) { - PMIX_INFO_FREE(p->info, p->ninfo); - } - if (NULL != p->bfrop) { - free(p->bfrop); - } - if (NULL != p->psec) { - free(p->psec); - } - if (NULL != p->cred) { - free(p->cred); - } -} -PMIX_CLASS_INSTANCE(pmix_pending_connection_t, - pmix_object_t, - pccon, pcdes); - static void prevcon(pmix_peer_events_info_t *p) { p->peer = NULL; @@ -1647,38 +1618,3 @@ static void regdes(pmix_regevents_info_t *p) PMIX_CLASS_INSTANCE(pmix_regevents_info_t, pmix_list_item_t, regcon, regdes); - -static void lcon(pmix_listener_t *p) -{ - memset(&p->address, 0, sizeof(struct sockaddr_un)); - p->address.sun_family = AF_UNIX; - p->socket = -1; - p->varname = NULL; - p->uri = NULL; - p->owner_given = false; - p->group_given = false; - p->mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH; -} -static void ldes(pmix_listener_t *p) -{ - if (0 <= p->socket) { - CLOSE_THE_SOCKET(p->socket); - } - /* cleanup the rendezvous file */ - if (0 == access(p->address.sun_path, F_OK)) { - unlink(p->address.sun_path); - } - if (NULL != p->varname) { - free(p->varname); - } - if (NULL != p->uri) { - free(p->uri); - } -} -PMIX_CLASS_INSTANCE(pmix_listener_t, - pmix_list_item_t, - lcon, ldes); - -PMIX_CLASS_INSTANCE(pmix_usock_queue_t, - pmix_object_t, - NULL, NULL); diff --git a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h index e86aeb2511..0531581e44 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h +++ b/opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h @@ -19,7 +19,6 @@ #include #include #include -#include "src/usock/usock.h" #include "src/util/hash.h" typedef struct { @@ -86,32 +85,6 @@ typedef struct { } pmix_dmdx_local_t; PMIX_CLASS_DECLARATION(pmix_dmdx_local_t); -/* define listener protocol types */ -typedef uint16_t pmix_listener_protocol_t; -#define PMIX_PROTOCOL_V1 0 -#define PMIX_PROTOCOL_TOOL 1 -#define PMIX_PROTOCOL_V2 2 - -/* connection support */ -typedef struct { - pmix_object_t super; - pmix_event_t ev; - pmix_listener_protocol_t protocol; - int sd; - char nspace[PMIX_MAX_NSLEN+1]; - pmix_info_t *info; - size_t ninfo; - pmix_status_t status; - struct sockaddr_storage addr; - char *bfrop; - char *psec; - pmix_bfrop_buffer_type_t buffer_type; - char *cred; - uid_t uid; - gid_t gid; -} pmix_pending_connection_t; -PMIX_CLASS_DECLARATION(pmix_pending_connection_t); - /* event/error registration book keeping */ typedef struct { pmix_list_item_t super; @@ -127,45 +100,17 @@ typedef struct { } pmix_regevents_info_t; PMIX_CLASS_DECLARATION(pmix_regevents_info_t); -/* listener objects */ -typedef struct pmix_listener_t { - pmix_list_item_t super; - pmix_listener_protocol_t protocol; - int socket; - struct sockaddr_un address; - char *varname; - char *uri; - uint32_t owner; - bool owner_given; - uint32_t group; - bool group_given; - uint32_t mode; -} pmix_listener_t; -PMIX_CLASS_DECLARATION(pmix_listener_t); - typedef struct { pmix_pointer_array_t clients; // array of pmix_peer_t local clients pmix_list_t collectives; // list of active pmix_server_trkr_t pmix_list_t remote_pnd; // list of pmix_dmdx_remote_t awaiting arrival of data fror servicing remote req's pmix_list_t local_reqs; // list of pmix_dmdx_local_t awaiting arrival of data from local neighbours - volatile bool listen_thread_active; // listen thread is running - pmix_list_t listeners; // list of pmix_listener_t - int stop_thread[2]; // pipe used to stop listener thread pmix_buffer_t gdata; // cache of data given to me for passing to all clients pmix_list_t events; // list of pmix_regevents_info_t registered events pmix_ring_buffer_t notifications; // ring buffer of pending notifications bool tool_connections_allowed; } pmix_server_globals_t; -typedef struct { - pmix_object_t super; - pmix_event_t ev; - pmix_peer_t *peer; - pmix_buffer_t *buf; - uint32_t tag; -} pmix_usock_queue_t; -PMIX_CLASS_DECLARATION(pmix_usock_queue_t); - #define PMIX_PEER_CADDY(c, p, t) \ do { \ (c) = PMIX_NEW(pmix_server_caddy_t); \ @@ -174,14 +119,6 @@ PMIX_CLASS_DECLARATION(pmix_usock_queue_t); (c)->peer = (p); \ } while (0) -#define PMIX_SND_CADDY(c, h, s) \ - do { \ - (c) = PMIX_NEW(pmix_server_caddy_t); \ - (void)memcpy(&(c)->hdr, &(h), sizeof(pmix_usock_hdr_t)); \ - PMIX_RETAIN((s)); \ - (c)->snd = (s); \ - } while (0) - #define PMIX_SETUP_COLLECTIVE(c, t) \ do { \ (c) = PMIX_NEW(pmix_trkr_caddy_t); \ @@ -197,33 +134,6 @@ PMIX_CLASS_DECLARATION(pmix_usock_queue_t); } while (0) -/* queue a message to be sent to one of our procs - must - * provide the following params: - * - * t - tag to be sent to - * b - buffer to be sent - */ -#define PMIX_SERVER_QUEUE_REPLY(p, t, b) \ - do { \ - pmix_usock_queue_t *queue; \ - queue = PMIX_NEW(pmix_usock_queue_t); \ - queue->peer = (p); \ - queue->buf = (b); \ - queue->tag = (t); \ - pmix_output_verbose(2, pmix_globals.debug_output, \ - "[%s:%d] queue reply to %s:%d on tag %d", \ - __FILE__, __LINE__, \ - (queue->peer)->info->nptr->nspace, \ - (queue->peer)->info->rank, (queue->tag)); \ - event_assign(&queue->ev, pmix_globals.evbase, -1, \ - EV_WRITE, pmix_server_queue_message, queue); \ - event_priority_set(&queue->ev, 0); \ - event_active(&queue->ev, EV_WRITE, 1); \ - } while (0) - -pmix_status_t pmix_prepare_listening(pmix_listener_t *lt, bool *need_listener); -pmix_status_t pmix_start_listening(void); -void pmix_stop_listening(void); bool pmix_server_trk_update(pmix_server_trkr_t *trk); @@ -305,8 +215,6 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer, void *cbdata); void pmix_server_execute_collective(int sd, short args, void *cbdata); -void pmix_server_queue_message(int fd, short args, void *cbdata); - extern pmix_server_module_t pmix_host_server; extern pmix_server_globals_t pmix_server_globals; diff --git a/opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c b/opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c index 4a50b57f67..0d4e4a2bc3 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c +++ b/opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c @@ -68,7 +68,7 @@ extern pmix_client_globals_t pmix_client_globals; #include "src/util/output.h" #include "src/runtime/pmix_progress_threads.h" #include "src/runtime/pmix_rte.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/ptl.h" #include "src/mca/psec/psec.h" #include "src/include/pmix_globals.h" #if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1) @@ -77,18 +77,14 @@ extern pmix_client_globals_t pmix_client_globals; #define PMIX_MAX_RETRIES 10 -static char *mytmpdir = NULL; -static char *systmpdir = NULL; - -static pmix_status_t usock_connect(struct sockaddr_un *address, int *fd); - static void _notify_complete(pmix_status_t status, void *cbdata) { pmix_event_chain_t *chain = (pmix_event_chain_t*)cbdata; PMIX_RELEASE(chain); } -static void pmix_tool_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr, +static void pmix_tool_notify_recv(struct pmix_peer_t *peer, + pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { pmix_status_t rc; @@ -154,55 +150,13 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *hd } - -static pmix_status_t connect_to_server(struct sockaddr_un *address) -{ - int sd; - pmix_status_t ret; - - if (PMIX_SUCCESS != (ret = usock_connect(address, &sd))) { - PMIX_ERROR_LOG(ret); - return ret; - } - pmix_client_globals.myserver.sd = sd; - /* setup recv event */ - event_assign(&pmix_client_globals.myserver.recv_event, - pmix_globals.evbase, - pmix_client_globals.myserver.sd, - EV_READ | EV_PERSIST, - pmix_usock_recv_handler, &pmix_client_globals.myserver); - event_add(&pmix_client_globals.myserver.recv_event, 0); - pmix_client_globals.myserver.recv_ev_active = true; - - /* setup send event */ - event_assign(&pmix_client_globals.myserver.send_event, - pmix_globals.evbase, - pmix_client_globals.myserver.sd, - EV_WRITE|EV_PERSIST, - pmix_usock_send_handler, &pmix_client_globals.myserver); - pmix_client_globals.myserver.send_ev_active = false; - - return PMIX_SUCCESS; -} - PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, pmix_info_t info[], size_t ninfo) { - char *evar, *tdir, *tmp; - struct sockaddr_un address; - size_t n; pmix_kval_t *kptr; pmix_status_t rc; pmix_nspace_t *nptr, *nsptr; - pid_t server_pid=0; - bool server_pid_given = false; - int hostnamelen = 30; - char hostname[hostnamelen]; - DIR *cur_dirp = NULL; - struct dirent * dir_entry; - bool connect_to_system_server = false; - bool connect_to_system_first = false; - bool connection_defined = false; + char hostname[PMIX_MAX_NSLEN]; if (NULL == proc) { return PMIX_ERR_BAD_PARAM; @@ -220,28 +174,6 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, return PMIX_SUCCESS; } - /* scan incoming info for directives */ - if (NULL != info) { - for (n=0; n < ninfo; n++) { - if (strcmp(info[n].key, PMIX_SERVER_PIDINFO) == 0) { - server_pid = info[n].value.data.pid; - server_pid_given = true; - } else if (strcmp(info[n].key, PMIX_CONNECT_TO_SYSTEM) == 0) { - connect_to_system_server = info[n].value.data.flag; - connection_defined = true; - } else if (strcmp(info[n].key, PMIX_CONNECT_SYSTEM_FIRST) == 0) { - connect_to_system_first = info[n].value.data.flag; - connection_defined = true; - } else if (strcmp(info[n].key, PMIX_SERVER_TMPDIR) == 0 && - NULL == mytmpdir) { - mytmpdir = strdup(info[n].value.data.string); - } else if (strcmp(info[n].key, PMIX_SYSTEM_TMPDIR) == 0 && - NULL == systmpdir) { - systmpdir = strdup(info[n].value.data.string); - } - } - } - /* setup the runtime - this init's the globals, * opens and initializes the required frameworks */ if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_TOOL, info, ninfo, @@ -256,112 +188,20 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, pmix_output_verbose(2, pmix_globals.debug_output, "pmix: init called"); - /* setup the path to the daemon rendezvous point */ - memset(&address, 0, sizeof(struct sockaddr_un)); - address.sun_family = AF_UNIX; - /* Get hostname to match what the server is doing */ - gethostname(hostname, hostnamelen); - /* ensure it is NULL terminated */ - hostname[hostnamelen-1] = '\0'; - - /* if we are to connect solely to the system-level daemon, - * or to preferentially connect to the system-level daemon, - * or nothing was specified at all, then look to see if a - * rendezvous point in that location exists */ - if (connect_to_system_server || connect_to_system_first || !connection_defined) { - /* find the temp dir */ - if (NULL != systmpdir) { - tdir = systmpdir; - } else if (NULL == (tdir = getenv("TMPDIR"))) { - if (NULL == (tdir = getenv("TEMP"))) { - if (NULL == (tdir = getenv("TMP"))) { - tdir = "/tmp"; - } - } - } - snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/pmix.sys.%s", tdir, hostname); - /* see if the rendezvous file exists */ - if (0 != access(address.sun_path, R_OK)) { - /* if it was a requirement, then error out */ - if (connect_to_system_server) { - return PMIX_ERR_UNREACH; - } - /* otherwise, this isn't a fatal error - reset the addr */ - memset(&address, 0, sizeof(struct sockaddr_un)); - address.sun_family = AF_UNIX; - connection_defined = false; - } else { - /* connect to this server */ - connection_defined = true; - } + /* select our psec module - we take the default as we cannot + * do any better */ + if (PMIX_SUCCESS != (rc = pmix_psec.assign_module(pmix_globals.mypeer, NULL))) { + return PMIX_ERR_INIT; } + /* the server will have to use the same */ + pmix_client_globals.myserver.compat.psec = pmix_globals.mypeer->compat.psec; - if (!connection_defined) { - /* if we get here, then either we are to connect to - * a non-system daemon, or a system-level daemon was - * not found - so now look for the session daemon */ - - - /* find the temp dir */ - if (NULL != mytmpdir) { - tdir = mytmpdir; - } else if (NULL == (tdir = getenv("TMPDIR"))) { - if (NULL == (tdir = getenv("TEMP"))) { - if (NULL == (tdir = getenv("TMP"))) { - tdir = "/tmp"; - } - } - } - - /* if they gave us a specific pid, then look for that - * particular server - otherwise, see if there is only - * one on this node and default to it */ - if (server_pid_given) { - snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/pmix.%s.%d", tdir, hostname, server_pid); - /* if the rendezvous file doesn't exist, that's an error */ - if (0 != access(address.sun_path, R_OK)) { - return PMIX_ERR_NOT_FOUND; - } - } else { - /* open up the temp directory */ - if (NULL == (cur_dirp = opendir(tdir))) { - return PMIX_ERR_NOT_FOUND; - } - /* search the entries for something that starts with pmix.hostname */ - if (0 > asprintf(&tmp, "pmix.%s", hostname)) { - closedir(cur_dirp); - return PMIX_ERR_NOMEM; - } - evar = NULL; - while (NULL != (dir_entry = readdir(cur_dirp))) { - if (0 == strncmp(dir_entry->d_name, tmp, strlen(tmp))) { - /* found one - if more than one, then that's an error */ - if (NULL != evar) { - closedir(cur_dirp); - free(evar); - free(tmp); - return PMIX_ERR_INIT; - } - evar = strdup(dir_entry->d_name); - } - } - free(tmp); - closedir(cur_dirp); - if (NULL == evar) { - /* none found */ - return PMIX_ERR_INIT; - } - /* use the found one as our contact point */ - snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/%s", tdir, evar); - free(evar); - } - } - - /* connect to the server */ - if (PMIX_SUCCESS != (rc = connect_to_server(&address))) { - PMIX_ERROR_LOG(rc); + /* connect to the server - returns job info if successful */ + if (PMIX_SUCCESS != (rc = pmix_ptl.connect_to_peer(&pmix_client_globals.myserver, info, ninfo))){ return rc; } + +pmix_output(0, "TOOL: SERVER CONNECTION COMPLETE"); /* increment our init reference counter */ pmix_globals.init_cntr++; @@ -381,6 +221,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, } } if (NULL == nsptr) { +pmix_output(0, "TOOL: NSPACE NOT FOUND"); return PMIX_ERR_NOT_FOUND; } @@ -632,275 +473,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) PMIX_DESTRUCT(&pmix_client_globals.myserver); PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests); - if (NULL != mytmpdir) { - free(mytmpdir); - } - if (NULL != systmpdir) { - free(systmpdir); - } - pmix_globals_finalize(); pmix_class_finalize(); return PMIX_SUCCESS; } - -/* - * The sections below need to be updated to reflect tool - * connection handshake protocols - in this case, we - * don't know our nspace/rank in advance. So we need - * the handshake to include the security credential - * exchange, and then get our nspace/rank in return */ - -static pmix_status_t send_connect_ack(int sd) -{ - char *msg; - pmix_usock_hdr_t hdr; - size_t sdsize=0, csize=0; - char *cred = NULL; - char *sec; - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: TOOL SEND CONNECT ACK"); - - /* setup the header */ - memset(&hdr, 0, sizeof(pmix_usock_hdr_t)); - hdr.pindex = -1; - hdr.tag = UINT32_MAX; - - /* get a credential, if the security system provides one. Not - * every SPC will do so, thus we must first check */ - if (NULL != pmix_globals.mypeer->compat.psec->create_cred) { - if (NULL == (cred = pmix_globals.mypeer->compat.psec->create_cred())) { - /* an error occurred - we cannot continue */ - return PMIX_ERR_INVALID_CRED; - } - csize = strlen(cred) + 1; // must NULL terminate the string! - } - - /* add our active bfrops and sec module info, and what type - * of buffers we are using */ - // bfrop = pmix_globals.mypeer->compat.bfrops->name; - sec = pmix_globals.mypeer->compat.psec->name; - - /* set the number of bytes to be read beyond the header */ - hdr.nbytes = strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 + csize; // must NULL terminate the VERSION string! - - /* create a space for our message */ - sdsize = (sizeof(hdr) + hdr.nbytes); - if (NULL == (msg = (char*)malloc(sdsize))) { - if (NULL != cred) { - free(cred); - } - return PMIX_ERR_OUT_OF_RESOURCE; - } - memset(msg, 0, sdsize); - - csize=0; - memcpy(msg, &hdr, sizeof(pmix_usock_hdr_t)); - csize += sizeof(pmix_usock_hdr_t); - - /* load the message */ - memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION)); - csize += strlen(PMIX_VERSION)+1; - // memcpy(msg+csize, bfrop, strlen(bfrop)); - // csize += strlen(bfrop)+1; - memcpy(msg+csize, sec, strlen(sec)); - csize += strlen(sec)+1; - // memcpy(msg+csize, &pmix_globals.mypeer->compat.type, sizeof(pmix_bfrop_buffer_type_t)); - // csize += sizeof(pmix_bfrop_buffer_type_t); - if (NULL != cred) { - memcpy(msg+csize, cred, strlen(cred)); // leaves last position in msg set to NULL - } - - if (PMIX_SUCCESS != pmix_usock_send_blocking(sd, msg, sdsize)) { - free(msg); - if (NULL != cred) { - free(cred); - } - return PMIX_ERR_UNREACH; - } - - free(msg); - if (NULL != cred) { - free(cred); - } - - return PMIX_SUCCESS; -} - - -/* we receive a connection acknowledgement from the server, - * consisting of the status and (if success) the nspace assigned - * to us */ -static pmix_status_t recv_connect_ack(int sd) -{ - pmix_status_t reply; - struct timeval tv, save; - pmix_socklen_t sz; - bool sockopt = true; - pmix_nspace_t *nsptr; - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: RECV CONNECT ACK FROM SERVER"); - - /* get the current timeout value so we can reset to it */ - sz = sizeof(save); - if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) { - if (ENOPROTOOPT == errno) { - sockopt = false; - } else { - return PMIX_ERR_UNREACH; - } - } else { - /* set a timeout on the blocking recv so we don't hang */ - tv.tv_sec = 2; - tv.tv_usec = 0; - if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO"); - return PMIX_ERR_UNREACH; - } - } - - /* get the returned status from the request for namespace */ - pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t)); - if (PMIX_SUCCESS != reply) { - return reply; - } - - /* get our assigned nspace */ - pmix_usock_recv_blocking(sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1); - - /* setup required bookkeeping */ - nsptr = PMIX_NEW(pmix_nspace_t); - (void)strncpy(nsptr->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); - pmix_list_append(&pmix_globals.nspaces, &nsptr->super); - /* our rank is always zero */ - pmix_globals.myid.rank = 0; - - /* get the server's nspace and rank so we can send to it */ - pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t); - pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t); - pmix_usock_recv_blocking(sd, (char*)pmix_client_globals.myserver.info->nptr->nspace, PMIX_MAX_NSLEN+1); - pmix_usock_recv_blocking(sd, (char*)&(pmix_client_globals.myserver.info->rank), sizeof(int)); - - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: RECV CONNECT CONFIRMATION FOR TOOL %s:%d FROM SERVER %s:%d", - pmix_globals.myid.nspace, pmix_globals.myid.rank, - pmix_client_globals.myserver.info->nptr->nspace, - pmix_client_globals.myserver.info->rank); - - /* get the returned status from the security handshake */ - pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t)); - if (PMIX_SUCCESS != reply) { - /* see if they want us to do the handshake */ - if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { - if (NULL == pmix_globals.mypeer->compat.psec->client_handshake) { - return PMIX_ERR_HANDSHAKE_FAILED; - } - if (PMIX_SUCCESS != (reply = pmix_globals.mypeer->compat.psec->client_handshake(sd))) { - return reply; - } - /* if the handshake succeeded, then fall thru to the next step */ - } else { - return reply; - } - } - - if (sockopt) { - if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) { - return PMIX_ERR_UNREACH; - } - } - - return PMIX_SUCCESS; -} - -static pmix_status_t usock_connect(struct sockaddr_un *addr, int *fd) -{ - int sd=-1; - pmix_status_t rc; - pmix_socklen_t addrlen = 0; - int retries = 0; - - pmix_output_verbose(2, pmix_globals.debug_output, - "usock_peer_try_connect: attempting to connect to server at %s", - addr->sun_path); - - addrlen = sizeof(struct sockaddr_un); - while (retries < PMIX_MAX_RETRIES) { - retries++; - /* Create the new socket */ - sd = socket(PF_UNIX, SOCK_STREAM, 0); - if (sd < 0) { - pmix_output(0, "pmix:create_socket: socket() failed: %s (%d)\n", - strerror(pmix_socket_errno), - pmix_socket_errno); - continue; - } - pmix_output_verbose(2, pmix_globals.debug_output, - "usock_peer_try_connect: attempting to connect to server on socket %d", sd); - /* try to connect */ - int err = -1; - if ((err = connect(sd, (struct sockaddr*)addr, addrlen)) < 0) { - if (pmix_socket_errno == ETIMEDOUT) { - /* The server may be too busy to accept new connections */ - pmix_output_verbose(2, pmix_globals.debug_output, - "timeout connecting to server"); - CLOSE_THE_SOCKET(sd); - continue; - } else if (ECONNABORTED == pmix_socket_errno) { - /* Some kernels (Linux 2.6) will automatically software - abort a connection that was ECONNREFUSED on the last - attempt, without even trying to establish the - connection. Handle that case in a semi-rational - way by trying twice before giving up */ - pmix_output_verbose(2, pmix_globals.debug_output, - "connection to server aborted by OS - retrying"); - CLOSE_THE_SOCKET(sd); - continue; - } else { - pmix_output_verbose(2, pmix_globals.debug_output, - "Failed to connect, errno = %d, err= %s\n", errno, strerror(errno)); - CLOSE_THE_SOCKET(sd); - continue; - } - } - /* otherwise, the connect succeeded - so break out of the loop */ - break; - } - - if (retries == PMIX_MAX_RETRIES || sd < 0){ - /* We were unsuccessful in establishing this connection, and are - * not likely to suddenly become successful */ - if (0 <= sd) { - CLOSE_THE_SOCKET(sd); - } - return PMIX_ERR_UNREACH; - } - - /* send any authentication credentials to the server */ - if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) { - PMIX_ERROR_LOG(rc); - CLOSE_THE_SOCKET(sd); - return rc; - } - - /* do whatever handshake is required */ - if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) { - PMIX_ERROR_LOG(rc); - CLOSE_THE_SOCKET(sd); - return rc; - } - - pmix_output_verbose(2, pmix_globals.debug_output, - "sock_peer_try_connect: Connection across to server succeeded"); - - /* mark the connection as made */ - pmix_globals.connected = true; - - pmix_usock_set_nonblocking(sd); - - *fd = sd; - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix2x/pmix/src/usock/Makefile.am b/opal/mca/pmix/pmix2x/pmix/src/usock/Makefile.am deleted file mode 100644 index a9026a2da8..0000000000 --- a/opal/mca/pmix/pmix2x/pmix/src/usock/Makefile.am +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright (c) 2014-2016 Intel, Inc. All rights reserved. -# Copyright (c) 2014 Artem Y. Polyakov . -# All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -headers += \ - usock/usock.h - -sources += \ - usock/usock.c \ - usock/usock_sendrecv.c diff --git a/opal/mca/pmix/pmix2x/pmix/src/usock/usock.c b/opal/mca/pmix/pmix2x/pmix/src/usock/usock.c deleted file mode 100644 index 6686ba1a17..0000000000 --- a/opal/mca/pmix/pmix2x/pmix/src/usock/usock.c +++ /dev/null @@ -1,348 +0,0 @@ -/* - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. - * Copyright (c) 2014 Artem Y. Polyakov . - * All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Mellanox Technologies, Inc. - * All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include - -#include -#include -#include - -#include "src/include/pmix_globals.h" - -#ifdef HAVE_STRING_H -#include -#endif -#include -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_SYS_UN_H -#include -#endif -#ifdef HAVE_SYS_UIO_H -#include -#endif -#ifdef HAVE_SYS_TYPES_H -#include -#endif - -#include "src/buffer_ops/buffer_ops.h" -#include "src/util/output.h" - -#include "usock.h" - -/* instance usock globals */ -pmix_usock_globals_t pmix_usock_globals = {{{0}}}; - -void pmix_usock_init(pmix_usock_cbfunc_t cbfunc) -{ - pmix_usock_posted_recv_t *req; - - /* setup the usock globals */ - PMIX_CONSTRUCT(&pmix_usock_globals.posted_recvs, pmix_list_t); - - /* if a cbfunc was given, post a persistent recv - * for the special 0 tag so the client can recv - * error notifications from the server */ - if (NULL != cbfunc) { - req = PMIX_NEW(pmix_usock_posted_recv_t); - req->tag = 0; - req->cbfunc = cbfunc; - pmix_output_verbose(5, pmix_globals.debug_output, - "posting notification recv on tag %d", req->tag); - /* add it to the list of recvs - we cannot have unexpected messages - * in this subsystem as the server never sends us something that - * we didn't previously request */ - pmix_list_prepend(&pmix_usock_globals.posted_recvs, &req->super); - } -} - -void pmix_usock_finalize(void) -{ - PMIX_LIST_DESTRUCT(&pmix_usock_globals.posted_recvs); -} - -pmix_status_t pmix_usock_set_nonblocking(int sd) -{ - int flags; - /* setup the socket as non-blocking */ - if ((flags = fcntl(sd, F_GETFL, 0)) < 0) { - pmix_output(0, "usock_peer_connect: fcntl(F_GETFL) failed: %s (%d)\n", - strerror(pmix_socket_errno), - pmix_socket_errno); - } else { - flags |= O_NONBLOCK; - if(fcntl(sd, F_SETFL, flags) < 0) - pmix_output(0, "usock_peer_connect: fcntl(F_SETFL) failed: %s (%d)\n", - strerror(pmix_socket_errno), - pmix_socket_errno); - } - return PMIX_SUCCESS; -} - -pmix_status_t pmix_usock_set_blocking(int sd) -{ - int flags; - /* setup the socket as non-blocking */ - if ((flags = fcntl(sd, F_GETFL, 0)) < 0) { - pmix_output(0, "usock_peer_connect: fcntl(F_GETFL) failed: %s (%d)\n", - strerror(pmix_socket_errno), - pmix_socket_errno); - } else { - flags &= ~(O_NONBLOCK); - if(fcntl(sd, F_SETFL, flags) < 0) - pmix_output(0, "usock_peer_connect: fcntl(F_SETFL) failed: %s (%d)\n", - strerror(pmix_socket_errno), - pmix_socket_errno); - } - return PMIX_SUCCESS; -} - -/* - * A blocking send on a non-blocking socket. Used to send the small amount of connection - * information that identifies the peers endpoint. - */ -pmix_status_t pmix_usock_send_blocking(int sd, char *ptr, size_t size) -{ - size_t cnt = 0; - int retval; - - pmix_output_verbose(8, pmix_globals.debug_output, - "send blocking of %"PRIsize_t" bytes to socket %d", - size, sd ); - while (cnt < size) { - retval = send(sd, (char*)ptr+cnt, size-cnt, 0); - if (retval < 0) { - if (EAGAIN == pmix_socket_errno || - EWOULDBLOCK == pmix_socket_errno) { - /* just cycle and let it try again */ - pmix_output_verbose(8, pmix_globals.debug_output, - "blocking_send received error %d:%s from remote - cycling", - pmix_socket_errno, strerror(pmix_socket_errno)); - continue; - } - if (pmix_socket_errno != EINTR) { - pmix_output_verbose(8, pmix_globals.debug_output, - "usock_peer_send_blocking: send() to socket %d failed: %s (%d)\n", - sd, strerror(pmix_socket_errno), - pmix_socket_errno); - return PMIX_ERR_UNREACH; - } - continue; - } - cnt += retval; - } - - pmix_output_verbose(8, pmix_globals.debug_output, - "blocking send complete to socket %d", sd); - return PMIX_SUCCESS; -} - -/* - * A blocking recv on a non-blocking socket. Used to receive the small amount of connection - * information that identifies the peers endpoint. - */ -pmix_status_t pmix_usock_recv_blocking(int sd, char *data, size_t size) -{ - size_t cnt = 0; - - pmix_output_verbose(8, pmix_globals.debug_output, - "waiting for blocking recv of %"PRIsize_t" bytes", size); - - while (cnt < size) { - int retval = recv(sd, (char *)data+cnt, size-cnt, MSG_WAITALL); - - /* remote closed connection */ - if (retval == 0) { - pmix_output_verbose(8, pmix_globals.debug_output, - "usock_recv_blocking: remote closed connection"); - return PMIX_ERR_UNREACH; - } - - /* handle errors */ - if (retval < 0) { - if (EAGAIN == pmix_socket_errno || - EWOULDBLOCK == pmix_socket_errno) { - /* just cycle and let it try again */ - pmix_output_verbose(8, pmix_globals.debug_output, - "blocking_recv received error %d:%s from remote - cycling", - pmix_socket_errno, strerror(pmix_socket_errno)); - continue; - } - if (pmix_socket_errno != EINTR ) { - /* If we overflow the listen backlog, it's - possible that even though we finished the three - way handshake, the remote host was unable to - transition the connection from half connected - (received the initial SYN) to fully connected - (in the listen backlog). We likely won't see - the failure until we try to receive, due to - timing and the like. The first thing we'll get - in that case is a RST packet, which receive - will turn into a connection reset by peer - errno. In that case, leave the socket in - CONNECT_ACK and propogate the error up to - recv_connect_ack, who will try to establish the - connection again */ - pmix_output_verbose(8, pmix_globals.debug_output, - "blocking_recv received error %d:%s from remote - aborting", - pmix_socket_errno, strerror(pmix_socket_errno)); - return PMIX_ERR_UNREACH; - } - continue; - } - cnt += retval; - } - - pmix_output_verbose(8, pmix_globals.debug_output, - "blocking receive complete from remote"); - return PMIX_SUCCESS; -} - -/*** INSTANTIATE INTERNAL CLASSES ***/ -static void scon(pmix_usock_send_t *p) -{ - memset(&p->hdr, 0, sizeof(pmix_usock_hdr_t)); - p->hdr.tag = UINT32_MAX; - p->hdr.nbytes = 0; - p->data = NULL; - p->hdr_sent = false; - p->sdptr = NULL; - p->sdbytes = 0; -} -static void sdes(pmix_usock_send_t *p) -{ - if (NULL != p->data) { - PMIX_RELEASE(p->data); - } -} -PMIX_CLASS_INSTANCE(pmix_usock_send_t, - pmix_list_item_t, - scon, sdes); - -static void rcon(pmix_usock_recv_t *p) -{ - memset(&p->hdr, 0, sizeof(pmix_usock_hdr_t)); - p->hdr.tag = UINT32_MAX; - p->hdr.nbytes = 0; - p->data = NULL; - p->hdr_recvd = false; - p->rdptr = NULL; - p->rdbytes = 0; -} -PMIX_CLASS_INSTANCE(pmix_usock_recv_t, - pmix_list_item_t, - rcon, NULL); - -static void prcon(pmix_usock_posted_recv_t *p) -{ - p->tag = UINT32_MAX; - p->cbfunc = NULL; - p->cbdata = NULL; -} -PMIX_CLASS_INSTANCE(pmix_usock_posted_recv_t, - pmix_list_item_t, - prcon, NULL); - -static void cbcon(pmix_cb_t *p) -{ - p->active = false; - p->checked = false; - PMIX_CONSTRUCT(&p->data, pmix_buffer_t); - p->cbfunc = NULL; - p->op_cbfunc = NULL; - p->value_cbfunc = NULL; - p->lookup_cbfunc = NULL; - p->spawn_cbfunc = NULL; - p->cbdata = NULL; - memset(p->nspace, 0, PMIX_MAX_NSLEN+1); - p->rank = -1; - p->key = NULL; - p->value = NULL; - p->procs = NULL; - p->info = NULL; - p->ninfo = 0; - p->nvals = 0; -} -static void cbdes(pmix_cb_t *p) -{ - PMIX_DESTRUCT(&p->data); -} -PMIX_CLASS_INSTANCE(pmix_cb_t, - pmix_list_item_t, - cbcon, cbdes); - - -static void srcon(pmix_usock_sr_t *p) -{ - p->peer = NULL; - p->bfr = NULL; - p->cbfunc = NULL; - p->cbdata = NULL; -} -PMIX_CLASS_INSTANCE(pmix_usock_sr_t, - pmix_object_t, - srcon, NULL); - -static void pcon(pmix_peer_t *p) -{ - p->info = NULL; - p->proc_cnt = 0; - p->server_object = NULL; - p->index = 0; - p->sd = -1; - p->send_ev_active = false; - p->recv_ev_active = false; - PMIX_CONSTRUCT(&p->send_queue, pmix_list_t); - p->send_msg = NULL; - p->recv_msg = NULL; - memset(&p->compat, 0, sizeof(p->compat)); -} -static void pdes(pmix_peer_t *p) -{ - if (0 <= p->sd) { - CLOSE_THE_SOCKET(p->sd); - } - if (p->send_ev_active) { - event_del(&p->send_event); - } - if (p->recv_ev_active) { - event_del(&p->recv_event); - } - - if (NULL != p->info) { - PMIX_RELEASE(p->info); - } - - PMIX_LIST_DESTRUCT(&p->send_queue); - if (NULL != p->send_msg) { - PMIX_RELEASE(p->send_msg); - } - if (NULL != p->recv_msg) { - PMIX_RELEASE(p->recv_msg); - } -} -PMIX_CLASS_INSTANCE(pmix_peer_t, - pmix_object_t, - pcon, pdes); - - -PMIX_CLASS_INSTANCE(pmix_timer_t, - pmix_object_t, - NULL, NULL); diff --git a/opal/mca/pmix/pmix2x/pmix/src/usock/usock.h b/opal/mca/pmix/pmix2x/pmix/src/usock/usock.h deleted file mode 100644 index c7bbc09c6d..0000000000 --- a/opal/mca/pmix/pmix2x/pmix/src/usock/usock.h +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2011 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2006-2013 Los Alamos National Security, LLC. - * All rights reserved. - * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2015 Artem Y. Polyakov . - * All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Mellanox Technologies, Inc. - * All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - */ - -#ifndef USOCK_H -#define USOCK_H - -#include - -#include -#include - -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_SYS_SOCKET_H -#include -#endif -#ifdef HAVE_SYS_UN_H -#include -#endif -#ifdef HAVE_SYS_UIO_H -#include -#endif -#ifdef HAVE_NET_UIO_H -#include -#endif -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#include PMIX_EVENT_HEADER - -#include "src/include/pmix_globals.h" -#include "src/buffer_ops/buffer_ops.h" -#include "src/class/pmix_hash_table.h" -#include "src/class/pmix_list.h" - - - -/* usock structure for tracking posted recvs */ -typedef struct { - pmix_list_item_t super; - pmix_event_t ev; - uint32_t tag; - pmix_usock_cbfunc_t cbfunc; - void *cbdata; -} pmix_usock_posted_recv_t; -PMIX_CLASS_DECLARATION(pmix_usock_posted_recv_t); - -/* usock struct for posting send/recv request */ -typedef struct { - pmix_object_t super; - pmix_event_t ev; - pmix_peer_t *peer; - pmix_buffer_t *bfr; - pmix_usock_cbfunc_t cbfunc; - void *cbdata; -} pmix_usock_sr_t; -PMIX_CLASS_DECLARATION(pmix_usock_sr_t); - -/* usock struct for tracking ops */ -typedef struct { - pmix_list_item_t super; - pmix_event_t ev; - volatile bool active; - bool checked; - int status; - pmix_status_t pstatus; - pmix_scope_t scope; - pmix_buffer_t data; - pmix_usock_cbfunc_t cbfunc; - pmix_op_cbfunc_t op_cbfunc; - pmix_value_cbfunc_t value_cbfunc; - pmix_lookup_cbfunc_t lookup_cbfunc; - pmix_spawn_cbfunc_t spawn_cbfunc; - pmix_evhdlr_reg_cbfunc_t errreg_cbfunc; - size_t errhandler_ref; - void *cbdata; - char nspace[PMIX_MAX_NSLEN+1]; - pmix_rank_t rank; - char *key; - pmix_value_t *value; - pmix_proc_t *procs; - pmix_info_t *info; - size_t ninfo; - size_t nvals; -} pmix_cb_t; -PMIX_CLASS_DECLARATION(pmix_cb_t); - -typedef struct { - pmix_object_t super; - pmix_event_t ev; - void *cbdata; -} pmix_timer_t; -PMIX_CLASS_DECLARATION(pmix_timer_t); - -/* internal convenience macros */ -#define PMIX_ACTIVATE_SEND_RECV(p, b, cb, d) \ - do { \ - int rc = -1; \ - pmix_usock_sr_t *ms; \ - pmix_output_verbose(5, pmix_globals.debug_output, \ - "[%s:%d] post send to server", \ - __FILE__, __LINE__); \ - ms = PMIX_NEW(pmix_usock_sr_t); \ - ms->peer = (p); \ - ms->bfr = (b); \ - ms->cbfunc = (cb); \ - ms->cbdata = (d); \ - rc = event_assign(&((ms)->ev), pmix_globals.evbase, -1, \ - EV_WRITE, pmix_usock_send_recv, (ms)); \ - pmix_output_verbose(10, pmix_globals.debug_output, \ - "event_assign returned %d", rc); \ - event_active(&((ms)->ev), EV_WRITE, 1); \ - } while (0) - -#define PMIX_ACTIVATE_POST_MSG(ms) \ - do { \ - pmix_output_verbose(5, pmix_globals.debug_output, \ - "[%s:%d] post msg", \ - __FILE__, __LINE__); \ - event_assign(&((ms)->ev), pmix_globals.evbase, -1, \ - EV_WRITE, pmix_usock_process_msg, (ms)); \ - event_active(&((ms)->ev), EV_WRITE, 1); \ - } while (0) - -#define CLOSE_THE_SOCKET(socket) \ - do { \ - if (0 <= socket) { \ - shutdown(socket, 2); \ - close(socket); \ - socket = -1; \ - } \ - } while (0) - - -#define PMIX_TIMER_EVENT(s, f, d) \ - do { \ - pmix_timer_t *tm; \ - struct timeval tv; \ - tm = PMIX_NEW(pmix_timer_t); \ - tm->cbdata = (d); \ - event_assign(&tm->ev, pmix_globals.evbase, -1, 0, (f), tm); \ - tv.tv_sec = (s); \ - tv.tv_usec = 0; \ - PMIX_OUTPUT_VERBOSE((1, pmix_globals.debug_output, \ - "defining timer event: %ld sec %ld usec at %s:%d", \ - (long)tv.tv_sec, (long)tv.tv_usec, \ - __FILE__, __LINE__)); \ - event_add(&tm->ev, &tv); \ - } while (0) - - -/* usock common variables */ -typedef struct { - pmix_list_t posted_recvs; // list of pmix_usock_posted_recv_t -} pmix_usock_globals_t; -extern pmix_usock_globals_t pmix_usock_globals; - -/* usock common functions */ -void pmix_usock_init(pmix_usock_cbfunc_t cbfunc); -void pmix_usock_finalize(void); -pmix_status_t pmix_usock_set_nonblocking(int sd); -pmix_status_t pmix_usock_set_blocking(int sd); -pmix_status_t pmix_usock_send_blocking(int sd, char *ptr, size_t size); -pmix_status_t pmix_usock_recv_blocking(int sd, char *data, size_t size); -void pmix_usock_send_recv(int sd, short args, void *cbdata); -void pmix_usock_send_handler(int sd, short flags, void *cbdata); -void pmix_usock_recv_handler(int sd, short flags, void *cbdata); -void pmix_usock_process_msg(int fd, short flags, void *cbdata); - -#endif // USOCK_H diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include b/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include index 7f4a7e1f33..e5bad0fe2e 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include +++ b/opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include @@ -46,8 +46,9 @@ headers += \ util/strnlen.h \ util/hash.h \ util/net.h \ + util/pif.h \ util/tsd.h \ - util/pif.h + util/parse_options.h sources += \ util/argv.c \ @@ -67,7 +68,8 @@ sources += \ util/getid.c \ util/hash.c \ util/net.c \ - util/pif.c + util/pif.c \ + util/parse_options.c libpmix_la_LIBADD += \ util/keyval/libpmixutilkeyval.la diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/error.c b/opal/mca/pmix/pmix2x/pmix/src/util/error.c index 0f051907e2..fae071ef56 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/error.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/error.c @@ -145,6 +145,10 @@ const char* PMIx_Error_string(pmix_status_t errnum) return "PMIX_ERR_VALUE_OUT_OF_BOUNDS"; case PMIX_ERR_NETWORK_NOT_PARSEABLE: return "PMIX_ERR_NETWORK_NOT_PARSEABLE"; + case PMIX_ERR_FILE_OPEN_FAILURE: + return "PMIX_ERR_FILE_OPEN_FAILURE"; + case PMIX_ERR_FILE_READ_FAILURE: + return "PMIX_ERR_FILE_READ_FAILURE"; case PMIX_ERR_PERM: return "PMIX_ERR_PERM"; case PMIX_SUCCESS: diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/error.h b/opal/mca/pmix/pmix2x/pmix/src/util/error.h index ecf1ccfe59..27baf19f13 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/error.h +++ b/opal/mca/pmix/pmix2x/pmix/src/util/error.h @@ -62,6 +62,8 @@ #define PMIX_ERR_PERM (PMIX_INTERNAL_ERR_BASE - 31) #define PMIX_ERR_OPERATION_IN_PROGRESS (PMIX_INTERNAL_ERR_BASE - 32) #define PMIX_ERR_NETWORK_NOT_PARSEABLE (PMIX_INTERNAL_ERR_BASE - 33) +#define PMIX_ERR_FILE_OPEN_FAILURE (PMIX_INTERNAL_ERR_BASE - 34) +#define PMIX_ERR_FILE_READ_FAILURE (PMIX_INTERNAL_ERR_BASE - 35) #define PMIX_ERROR_LOG(r) \ do { \ diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/parse_options.c b/opal/mca/pmix/pmix2x/pmix/src/util/parse_options.c new file mode 100644 index 0000000000..d7679af6ac --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/util/parse_options.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2008 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "pmix_config.h" +#include "pmix_common.h" + +#include +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#include "src/util/argv.h" +#include "src/util/output.h" + +#include "src/util/parse_options.h" + +void pmix_util_parse_range_options(char *inp, char ***output) +{ + char **r1=NULL, **r2=NULL; + int i, vint; + int start, end, n; + char nstr[32]; + char *input, *bang; + bool bang_option=false; + + /* protect against null input */ + if (NULL == inp) { + return; + } + + /* protect the provided input */ + input = strdup(inp); + + /* check for the special '!' operator */ + if (NULL != (bang = strchr(input, '!'))) { + bang_option = true; + *bang = '\0'; + } + + /* split on commas */ + r1 = pmix_argv_split(input, ','); + /* for each resulting element, check for range */ + for (i=0; i < pmix_argv_count(r1); i++) { + r2 = pmix_argv_split(r1[i], '-'); + if (1 < pmix_argv_count(r2)) { + /* given range - get start and end */ + start = strtol(r2[0], NULL, 10); + end = strtol(r2[1], NULL, 10); + } else { + /* check for wildcard - have to do this here because + * the -1 would have been caught in the split + */ + vint = strtol(r1[i], NULL, 10); + if (-1 == vint) { + pmix_argv_free(*output); + *output = NULL; + pmix_argv_append_nosize(output, "-1"); + pmix_argv_free(r2); + goto cleanup; + } + start = strtol(r2[0], NULL, 10); + end = start; + } + for (n = start; n <= end; n++) { + snprintf(nstr, 32, "%d", n); + pmix_argv_append_nosize(output, nstr); + } + pmix_argv_free(r2); + } + +cleanup: + if (bang_option) { + pmix_argv_append_nosize(output, "BANG"); + } + free(input); + pmix_argv_free(r1); + +} + +void pmix_util_get_ranges(char *inp, char ***startpts, char ***endpts) +{ + char **r1=NULL, **r2=NULL; + int i; + char *input; + + /* protect against null input */ + if (NULL == inp) { + return; + } + + /* protect the provided input */ + input = strdup(inp); + + /* split on commas */ + r1 = pmix_argv_split(input, ','); + /* for each resulting element, check for range */ + for (i=0; i < pmix_argv_count(r1); i++) { + r2 = pmix_argv_split(r1[i], '-'); + if (2 == pmix_argv_count(r2)) { + /* given range - get start and end */ + pmix_argv_append_nosize(startpts, r2[0]); + pmix_argv_append_nosize(endpts, r2[1]); + } else if (1 == pmix_argv_count(r2)) { + /* only one value provided, so it is both the start + * and the end + */ + pmix_argv_append_nosize(startpts, r2[0]); + pmix_argv_append_nosize(endpts, r2[0]); + } else { + /* no idea how to parse this */ + pmix_output(0, "Unknown parse error on string: %s(%s)", inp, r1[i]); + } + pmix_argv_free(r2); + } + + free(input); + pmix_argv_free(r1); + +} diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/parse_options.h b/opal/mca/pmix/pmix2x/pmix/src/util/parse_options.h new file mode 100644 index 0000000000..67e239052d --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/src/util/parse_options.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** @file: + * + */ + +#ifndef _PMIX_PARSE_OPTIONS_H_ +#define _PMIX_PARSE_OPTIONS_H_ + +#include "pmix_config.h" + +BEGIN_C_DECLS + +PMIX_EXPORT void pmix_util_parse_range_options(char *input, char ***output); + +PMIX_EXPORT void pmix_util_get_ranges(char *inp, char ***startpts, char ***endpts); + +END_C_DECLS +#endif diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/pif.c b/opal/mca/pmix/pmix2x/pmix/src/util/pif.c index e6e52af89d..ab594e033a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/pif.c +++ b/opal/mca/pmix/pmix2x/pmix/src/util/pif.c @@ -269,10 +269,11 @@ int16_t pmix_ifaddrtokindex(const char* if_addr) for (r = res; r != NULL; r = r->ai_next) { PMIX_LIST_FOREACH(intf, &pmix_if_list, pmix_pif_t) { if (AF_INET == r->ai_family && AF_INET == intf->af_family) { - struct sockaddr_in ipv4; + struct sockaddr ipv4; + memset(&ipv4, 0, sizeof(struct sockaddr)); len = (r->ai_addrlen < sizeof(struct sockaddr_in)) ? r->ai_addrlen : sizeof(struct sockaddr_in); memcpy(&ipv4, r->ai_addr, len); - if (pmix_net_samenetwork((struct sockaddr*)&ipv4, (struct sockaddr*)&intf->if_addr, intf->if_mask)) { + if (pmix_net_samenetwork(&ipv4, (struct sockaddr*)&intf->if_addr, intf->if_mask)) { if_kernel_index = intf->if_kernel_index; freeaddrinfo (res); return if_kernel_index; diff --git a/opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h b/opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h index 15f334f792..ec2401e346 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h +++ b/opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h @@ -19,11 +19,11 @@ #if defined(HAVE_STRNLEN) #define PMIX_STRNLEN(c, a, b) \ - (c) = (int)strnlen(a, b) + (c) = strnlen(a, b) #else #define PMIX_STRNLEN(c, a, b) \ do { \ - int _x; \ + size_t _x; \ (c) = 0; \ for (_x=0; _x < (b); _x++) { \ if ('\0' == (a)[_x]) { \ diff --git a/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c index 4a5c733222..cb92364d0e 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c +++ b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.c @@ -60,7 +60,7 @@ void cli_connect(cli_info_t *cli, int sd, struct event_base * ebase, event_callb cli->ev = event_new(ebase, sd, EV_READ|EV_PERSIST, callback, cli); event_add(cli->ev,NULL); - pmix_usock_set_nonblocking(sd); + pmix_ptl_base_set_nonblocking(sd); TEST_VERBOSE(("Connection accepted from rank %d", cli_rank(cli) )); cli->state = CLI_CONNECTED; } diff --git a/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h index 173a384797..7f449fd4ae 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h +++ b/opal/mca/pmix/pmix2x/pmix/test/cli_stages.h @@ -25,7 +25,7 @@ #include "src/include/pmix_globals.h" #include "pmix_server.h" #include "src/class/pmix_list.h" -#include "src/usock/usock.h" +#include "src/mca/ptl/base/base.h" #include "test_common.h" diff --git a/opal/mca/pmix/pmix2x/pmix/test/pmix_client_otheruser.sh b/opal/mca/pmix/pmix2x/pmix/test/pmix_client_otheruser.sh old mode 100644 new mode 100755 diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c index 94039b926d..c345e94a85 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c @@ -35,12 +35,18 @@ #include "src/buffer_ops/types.h" #include "src/util/output.h" #include "src/util/printf.h" -#include "src/usock/usock.h" static uint32_t nprocs; static pmix_proc_t myproc; static uint32_t getcount = 0; +#define PMIX_WAIT_FOR_COMPLETION(a) \ + do { \ + while ((a)) { \ + usleep(10); \ + } \ + } while (0) + static void opcbfunc(pmix_status_t status, void *cbdata) { bool *active = (bool*)cbdata; diff --git a/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c b/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c index 74b5241e3d..528139e762 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c +++ b/opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c @@ -44,7 +44,6 @@ #include "src/util/printf.h" #include "src/util/argv.h" #include "src/buffer_ops/buffer_ops.h" -#include "src/usock/usock.h" static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata); @@ -209,7 +208,9 @@ int main(int argc, char **argv) myxfer_t *x; pmix_proc_t proc; wait_tracker_t *child; - pmix_info_t info; + pmix_info_t info[2]; + bool cross_version = false; + bool usock = true; /* smoke test */ if (PMIX_SUCCESS != 0) { @@ -219,14 +220,57 @@ int main(int argc, char **argv) fprintf(stderr, "Testing version %s\n", PMIx_Get_version()); + /* see if we were passed the number of procs to run or + * the executable to use */ + for (n=1; n < argc; n++) { + if (0 == strcmp("-n", argv[n]) && + NULL != argv[n+1]) { + nprocs = strtol(argv[n+1], NULL, 10); + ++n; // step over the argument + } else if (0 == strcmp("-e", argv[n]) && + NULL != argv[n+1]) { + executable = strdup(argv[n+1]); + for (k=n+2; NULL != argv[k]; k++) { + pmix_argv_append_nosize(&client_argv, argv[k]); + } + n += k; + } else if (0 == strcmp("-x", argv[n])) { + /* cross-version test - we will set one child to + * run at a different version. Requires -n >= 2 */ + cross_version = true; + usock = false; + } else if (0 == strcmp("-u", argv[n])) { + /* enable usock */ + usock = false; + } else if (0 == strcmp("-h", argv[n])) { + /* print the options and exit */ + fprintf(stderr, "usage: simptest \n"); + fprintf(stderr, " -n N Number of clients to run\n"); + fprintf(stderr, " -e foo Name of the client executable to run (default: simpclient\n"); + fprintf(stderr, " -x Test cross-version support\n"); + fprintf(stderr, " -u Enable legacy usock support\n"); + exit(0); + } + } + if (NULL == executable) { + executable = strdup("./simpclient"); + } + if (cross_version && nprocs < 2) { + fprintf(stderr, "Cross-version testing requires at least two clients\n"); + exit(1); + } + /* setup the server library and tell it to support tool connections */ - PMIX_INFO_CONSTRUCT(&info); - (void)strncpy(info.key, PMIX_SERVER_TOOL_SUPPORT, PMIX_MAX_KEYLEN); - if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, &info, 1))) { + PMIX_INFO_CONSTRUCT(&info[0]); + (void)strncpy(info[0].key, PMIX_SERVER_TOOL_SUPPORT, PMIX_MAX_KEYLEN); + PMIX_INFO_CONSTRUCT(&info[1]); + PMIX_INFO_LOAD(&info[1], PMIX_USOCK_DISABLE, &usock, PMIX_BOOL); + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, 2))) { fprintf(stderr, "Init failed with error %d\n", rc); return rc; } - PMIX_INFO_DESTRUCT(&info); + PMIX_INFO_DESTRUCT(&info[0]); + PMIX_INFO_DESTRUCT(&info[1]); /* register the errhandler */ PMIx_Register_event_handler(NULL, 0, NULL, 0, @@ -241,26 +285,6 @@ int main(int argc, char **argv) EV_SIGNAL|EV_PERSIST,wait_signal_callback, &handler); event_add(&handler, NULL); - /* see if we were passed the number of procs to run or - * the executable to use */ - for (n=1; n < (argc-1); n++) { - if (0 == strcmp("-n", argv[n]) && - NULL != argv[n+1]) { - nprocs = strtol(argv[n+1], NULL, 10); - ++n; // step over the argument - } else if (0 == strcmp("-e", argv[n]) && - NULL != argv[n+1]) { - executable = strdup(argv[n+1]); - for (k=n+2; NULL != argv[k]; k++) { - pmix_argv_append_nosize(&client_argv, argv[k]); - } - n += k; - } - } - if (NULL == executable) { - executable = strdup("./simpclient"); - } - /* we have a single namespace for all clients */ atmp = NULL; for (n=0; n < nprocs; n++) { @@ -296,10 +320,19 @@ int main(int argc, char **argv) PMIx_server_finalize(); return rc; } + /* if cross-version test is requested, then oscillate PTL support + * by rank */ + if (cross_version) { + if (0 == n % 2) { + pmix_setenv("PMIX_MCA_ptl", "tcp", true, &client_env); + } else { + pmix_setenv("PMIX_MCA_ptl", "usock", true, &client_env); + } + } x = PMIX_NEW(myxfer_t); if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, opcbfunc, x))) { - fprintf(stderr, "Server fork setup failed with error %d\n", rc); + fprintf(stderr, "Server register client failed with error %d\n", rc); PMIx_server_finalize(); return rc; } @@ -425,9 +458,8 @@ static pmix_status_t connected(const pmix_proc_t *proc, void *server_object, static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object, pmix_op_cbfunc_t cbfunc, void *cbdata) { - pmix_output(0, "SERVER: FINALIZED %s:%d", - proc->nspace, proc->rank); - --wakeup; + pmix_output(0, "SERVER: FINALIZED %s:%d WAKEUP %d", + proc->nspace, proc->rank, wakeup); /* ensure we call the cbfunc so the proc can exit! */ if (NULL != cbfunc) { cbfunc(PMIX_SUCCESS, cbdata); diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_cd.c b/opal/mca/pmix/pmix2x/pmix/test/test_cd.c index c078595eb5..e8cdb81905 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_cd.c +++ b/opal/mca/pmix/pmix2x/pmix/test/test_cd.c @@ -16,6 +16,13 @@ typedef struct { int status; } cd_cbdata; +#define PMIX_WAIT_FOR_COMPLETION(a) \ + do { \ + while ((a)) { \ + usleep(10); \ + } \ + } while (0) + static void cd_cb(pmix_status_t status, void *cbdata) { cd_cbdata *cb = (cd_cbdata*)cbdata; diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_common.h b/opal/mca/pmix/pmix2x/pmix/test/test_common.h index 12c02f8766..ca571c46f9 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_common.h +++ b/opal/mca/pmix/pmix2x/pmix/test/test_common.h @@ -28,7 +28,6 @@ #include "src/class/pmix_list.h" #include "src/util/argv.h" -#include "src/usock/usock.h" #define TEST_NAMESPACE "smoky_nspace" #define TEST_CREDENTIAL "dummy" diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_spawn.c b/opal/mca/pmix/pmix2x/pmix/test/test_spawn.c index d230b9a77c..da1922d4c8 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_spawn.c +++ b/opal/mca/pmix/pmix2x/pmix/test/test_spawn.c @@ -18,6 +18,13 @@ typedef struct { char nspace[PMIX_MAX_NSLEN]; } spawn_cbdata; +#define PMIX_WAIT_FOR_COMPLETION(a) \ + do { \ + while ((a)) { \ + usleep(10); \ + } \ + } while (0) + static void spawn_cb(pmix_status_t status, char nspace[], void *cbdata) { diff --git a/opal/mca/pmix/pmix_types.h b/opal/mca/pmix/pmix_types.h index f3edda91c9..4d79232150 100644 --- a/opal/mca/pmix/pmix_types.h +++ b/opal/mca/pmix/pmix_types.h @@ -62,8 +62,18 @@ BEGIN_C_DECLS #define OPAL_PMIX_GRPID "pmix.egid" // (uint32_t) effective group id /* attributes for the rendezvous socket */ +#define OPAL_PMIX_USOCK_DISABLE "pmix.usock.disable" // (bool) disable legacy usock support #define OPAL_PMIX_SOCKET_MODE "pmix.sockmode" // (uint32_t) POSIX mode_t (9 bits valid) +/* attributes for TCP connections */ +#define OPAL_PMIX_TCP_URI "pmix.tcp.uri" // (char*) URI of server to connect to +#define OPAL_PMIX_TCP_IF_INCLUDE "pmix.tcp.ifinclude" // (char*) comma-delimited list of devices and/or CIDR notation +#define OPAL_PMIX_TCP_IF_EXCLUDE "pmix.tcp.ifexclude" // (char*) comma-delimited list of devices and/or CIDR notation +#define OPAL_PMIX_TCP_IPV4_PORT "pmix.tcp.ipv4" // (int) IPv4 port to be used +#define OPAL_PMIX_TCP_IPV6_PORT "pmix.tcp.ipv6" // (int) IPv6 port to be used +#define OPAL_PMIX_TCP_DISABLE_IPV4 "pmix.tcp.disipv4" // (bool) true to disable IPv4 family +#define OPAL_PMIX_TCP_DISABLE_IPV6 "pmix.tcp.disipv6" // (bool) true to disable IPv6 family + /* general proc-level attributes */ #define OPAL_PMIX_CPUSET "pmix.cpuset" // (char*) hwloc bitmap applied to proc upon launch #define OPAL_PMIX_CREDENTIAL "pmix.cred" // (char*) security credential assigned to proc @@ -123,6 +133,8 @@ BEGIN_C_DECLS /* request-related info */ #define OPAL_PMIX_COLLECT_DATA "pmix.collect" // (bool) collect data and return it at the end of the operation #define OPAL_PMIX_TIMEOUT "pmix.timeout" // (int) time in sec before specified operation should time out +#define OPAL_PMIX_IMMEDIATE "pmix.immediate" // (bool) specified operation should immediately return an error if requested + // data cannot be found - do not request it from the host RM #define OPAL_PMIX_WAIT "pmix.wait" // (int) caller requests that the server wait until at least the specified // #values are found (0 => all and is the default) #define OPAL_PMIX_COLLECTIVE_ALGO "pmix.calgo" // (char*) comma-delimited list of algorithms to use for collective diff --git a/orte/mca/ess/singleton/ess_singleton_module.c b/orte/mca/ess/singleton/ess_singleton_module.c index 7ba5699725..7e523219cd 100644 --- a/orte/mca/ess/singleton/ess_singleton_module.c +++ b/orte/mca/ess/singleton/ess_singleton_module.c @@ -633,7 +633,7 @@ static int fork_hnp(void) orte_process_info.my_hnp_uri = orted_uri; /* split the pmix_uri into its parts */ - argv = opal_argv_split(cptr, ','); + argv = opal_argv_split(cptr, '*'); count = opal_argv_count(argv); /* push each piece into the environment */ for (i=0; i < count; i++) { diff --git a/orte/orted/orted_main.c b/orte/orted/orted_main.c index 65784f6a71..0c4d928c3b 100644 --- a/orte/orted/orted_main.c +++ b/orte/orted/orted_main.c @@ -612,12 +612,13 @@ int orte_daemon(int argc, char *argv[]) opal_argv_append_nosize(&singenv, env_str); free(env_str); - nptr = opal_argv_join(singenv, ','); + nptr = opal_argv_join(singenv, '*'); opal_argv_free(singenv); + /* create a string that contains our uri + sysinfo + PMIx server URI envars */ orte_util_convert_sysinfo_to_string(&sysinfo, orte_local_cpu_type, orte_local_cpu_model); asprintf(&tmp, "%s[%s]%s", orte_process_info.my_daemon_uri, sysinfo, nptr); - free(sysinfo); + free(sysinfo); free(nptr); /* pass that info to the singleton */ diff --git a/orte/orted/pmix/pmix_server.c b/orte/orted/pmix/pmix_server.c index a30c44c82f..c766ff584e 100644 --- a/orte/orted/pmix/pmix_server.c +++ b/orte/orted/pmix/pmix_server.c @@ -272,6 +272,12 @@ int pmix_server_init(void) kv->type = OPAL_STRING; kv->data.string = strdup(orte_process_info.tmpdir_base); opal_list_append(&info, &kv->super); + /* disable usock */ + kv = OBJ_NEW(opal_value_t); + kv->key = strdup(OPAL_PMIX_USOCK_DISABLE); + kv->type = OPAL_BOOL; + kv->data.flag = true; + opal_list_append(&info, &kv->super); /* setup the local server */ if (ORTE_SUCCESS != (rc = opal_pmix.server_init(&pmix_server, &info))) {