ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Update to latest PMIx master + PTL branch

Просмотр исходного кода 
Update the usock component to disable it

Signed-off-by: Ralph Castain <rhc@open-mpi.org>
Этот коммит содержится в:
Ralph Castain 2016-12-06 15:05:34 -08:00
родитель f95f11d285
Коммит fbed2d794a
79 изменённых файлов: 6043 добавлений и 3130 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
opal/mca/pmix/pmix2x/pmix/VERSION
Просмотреть файл

@ -30,7 +30,7 @@ greek=
# command, or with the date (if "git describe" fails) in the form of
# "date<date>".
repo_rev=gitc8f26f9
repo_rev=git22b754e
# If tarball_version is not empty, it is used as the version string in
# the tarball filename, regardless of all other versions listed in
@ -44,7 +44,7 @@ tarball_version=
# The date when this release was created
date="Dec 02, 2016"
date="Dec 06, 2016"
# The shared library version of each of PMIx's public libraries.
# These versions are maintained in accordance with the "Library

24
opal/mca/pmix/pmix2x/pmix/include/pmix_common.h
Просмотреть файл

@ -109,13 +109,15 @@ typedef uint32_t pmix_rank_t;
// of the internal progress thread
#define PMIX_SERVER_TOOL_SUPPORT "pmix.srvr.tool" // (bool) The host RM wants to declare itself as willing to
// accept tool connection requests
#define PMIX_SERVER_REMOTE_CONNECTIONS "pmix.srvr.remote" // (bool) Allow connections from remote tools (do not use loopback device)
#define PMIX_SERVER_SYSTEM_SUPPORT "pmix.srvr.sys" // (bool) The host RM wants to declare itself as being the local
// system server for PMIx connection requests
#define PMIX_SERVER_PIDINFO "pmix.srvr.pidinfo" // (pid_t) pid of the target server
#define PMIX_SERVER_HOSTNAME "pmix.srvr.host" // (char*) node where target server is located
#define PMIX_SERVER_TMPDIR "pmix.srvr.tmpdir" // (char*) temp directory where PMIx server will place
// client rendezvous points
// client rendezvous points and contact info
#define PMIX_SYSTEM_TMPDIR "pmix.sys.tmpdir" // (char*) temp directory for this system, where PMIx
// server will place tool rendezvous points
// server will place tool rendezvous points and contact info
#define PMIX_CONNECT_TO_SYSTEM "pmix.cnct.sys" // (bool) The requestor requires that a connection be made only to
// a local system-level PMIx server
#define PMIX_CONNECT_SYSTEM_FIRST "pmix.cnct.sys.first" // (bool) Preferentially look for a system-level PMIx server first
@ -124,10 +126,22 @@ typedef uint32_t pmix_rank_t;
#define PMIX_USERID "pmix.euid" // (uint32_t) effective user id
#define PMIX_GRPID "pmix.egid" // (uint32_t) effective group id
#define PMIX_DSTPATH "pmix.dstpath" // (char*) path to dstore files
#define PMIX_VERSION_INFO "pmix.version" // (char*) PMIx version of contactor
/* attributes for the rendezvous socket */
/* attributes for the USOCK rendezvous socket */
#define PMIX_USOCK_DISABLE "pmix.usock.disable" // (bool) disable legacy usock support
#define PMIX_SOCKET_MODE "pmix.sockmode" // (uint32_t) POSIX mode_t (9 bits valid)
/* attributes for TCP connections */
#define PMIX_TCP_URI "pmix.tcp.uri" // (char*) URI of server to connect to
#define PMIX_TCP_IF_INCLUDE "pmix.tcp.ifinclude" // (char*) comma-delimited list of devices and/or CIDR notation
#define PMIX_TCP_IF_EXCLUDE "pmix.tcp.ifexclude" // (char*) comma-delimited list of devices and/or CIDR notation
#define PMIX_TCP_IPV4_PORT "pmix.tcp.ipv4" // (int) IPv4 port to be used
#define PMIX_TCP_IPV6_PORT "pmix.tcp.ipv6" // (int) IPv6 port to be used
#define PMIX_TCP_DISABLE_IPV4 "pmix.tcp.disipv4" // (bool) true to disable IPv4 family
#define PMIX_TCP_DISABLE_IPV6 "pmix.tcp.disipv6" // (bool) true to disable IPv6 family
/* general proc-level attributes */
#define PMIX_CPUSET "pmix.cpuset" // (char*) hwloc bitmap applied to proc upon launch
#define PMIX_CREDENTIAL "pmix.cred" // (char*) security credential assigned to proc
@ -267,6 +281,10 @@ typedef uint32_t pmix_rank_t;
#define PMIX_LOG_STDOUT "pmix.log.stdout" // (bool) log data to stdout
#define PMIX_LOG_SYSLOG "pmix.log.syslog" // (bool) log data to syslog - defaults to ERROR priority unless
// modified by directive
/* debugger attributes */
#define PMIX_SPAWN_UNDER_DEBUGGER "pmix.dbg.pause" // (bool) job is being spawned under debugger - instruct it to pause on start
#define PMIX_JOB_BEING_DEBUGGED "pmix.dbg.job" // (char*) nspace of the job to be debugged - the RM/PMIx server are
// to provide the job-level info of that job to each debugger daemon
/**** PROCESS STATE DEFINITIONS ****/
typedef uint8_t pmix_proc_state_t;

1
opal/mca/pmix/pmix2x/pmix/src/Makefile.am
Просмотреть файл

@ -72,7 +72,6 @@ include runtime/Makefile.include
include tool/Makefile.include
include common/Makefile.include
include buffer_ops/Makefile.am
include usock/Makefile.am
if WANT_DSTORE
include sm/Makefile.include
include dstore/Makefile.include

431
opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c
Просмотреть файл

@ -65,7 +65,7 @@
#include "src/util/output.h"
#include "src/runtime/pmix_progress_threads.h"
#include "src/runtime/pmix_rte.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#include "src/include/pmix_globals.h"
#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
#include "src/dstore/pmix_dstore.h"
@ -76,15 +76,14 @@
#define PMIX_MAX_RETRIES 10
static pmix_status_t usock_connect(struct sockaddr *address, int *fd);
static void _notify_complete(pmix_status_t status, void *cbdata)
{
pmix_event_chain_t *chain = (pmix_event_chain_t*)cbdata;
PMIX_RELEASE(chain);
}
static void pmix_client_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr,
static void pmix_client_notify_recv(struct pmix_peer_t *peer,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_status_t rc;
@ -161,20 +160,21 @@ static void pmix_client_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *
pmix_client_globals_t pmix_client_globals = {{{0}}};
/* callback for wait completion */
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_cb_t *cb = (pmix_cb_t*)cbdata;
volatile bool *active = (volatile bool*)cbdata;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:client recv callback activated with %d bytes",
(NULL == buf) ? -1 : (int)buf->bytes_used);
"pmix:client wait_cbfunc received");
cb->active = false;
*active = false;
}
/* callback to receive job info */
static void job_data(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void job_data(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_status_t rc;
@ -201,49 +201,6 @@ static void job_data(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
cb->active = false;
}
static pmix_status_t connect_to_server(struct sockaddr_un *address, void *cbdata)
{
int sd;
pmix_status_t ret;
pmix_cmd_t cmd = PMIX_REQ_CMD;
pmix_buffer_t *req;
if (PMIX_SUCCESS != (ret=usock_connect((struct sockaddr *)address, &sd))) {
PMIX_ERROR_LOG(ret);
return ret;
}
pmix_client_globals.myserver.sd = sd;
/* setup recv event */
event_assign(&pmix_client_globals.myserver.recv_event,
pmix_globals.evbase,
pmix_client_globals.myserver.sd,
EV_READ | EV_PERSIST,
pmix_usock_recv_handler, &pmix_client_globals.myserver);
event_add(&pmix_client_globals.myserver.recv_event, 0);
pmix_client_globals.myserver.recv_ev_active = true;
/* setup send event */
event_assign(&pmix_client_globals.myserver.send_event,
pmix_globals.evbase,
pmix_client_globals.myserver.sd,
EV_WRITE|EV_PERSIST,
pmix_usock_send_handler, &pmix_client_globals.myserver);
pmix_client_globals.myserver.send_ev_active = false;
/* send a request for our job info - we do this as a non-blocking
* transaction because some systems cannot handle very large
* blocking operations and error out if we try them. */
req = PMIX_NEW(pmix_buffer_t);
if (PMIX_SUCCESS != (ret = pmix_bfrop.pack(req, &cmd, 1, PMIX_CMD))) {
PMIX_ERROR_LOG(ret);
PMIX_RELEASE(req);
return ret;
}
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, req, job_data, cbdata);
return PMIX_SUCCESS;
}
PMIX_EXPORT const char* PMIx_Get_version(void)
{
return pmix_version_string;
@ -252,11 +209,12 @@ PMIX_EXPORT const char* PMIx_Get_version(void)
PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
pmix_info_t info[], size_t ninfo)
{
char **uri, *evar;
int rc;
struct sockaddr_un address;
char *evar;
pmix_status_t rc;
pmix_nspace_t *nsptr;
pmix_cb_t cb;
pmix_buffer_t *req;
pmix_cmd_t cmd = PMIX_REQ_CMD;
if (NULL == proc) {
return PMIX_ERR_BAD_PARAM;
@ -293,7 +251,7 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: init called");
/* we require the nspace */
/* we require our nspace */
if (NULL == (evar = getenv("PMIX_NAMESPACE"))) {
/* let the caller know that the server isn't available yet */
return PMIX_ERR_INVALID_NAMESPACE;
@ -306,37 +264,6 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
(void)strncpy(nsptr->nspace, evar, PMIX_MAX_NSLEN);
pmix_list_append(&pmix_globals.nspaces, &nsptr->super);
/* if we don't have a path to the daemon rendezvous point,
* then we need to return an error */
if (NULL == (evar = getenv("PMIX_SERVER_URI"))) {
/* let the caller know that the server isn't available */
return PMIX_ERR_SERVER_NOT_AVAIL;
}
uri = pmix_argv_split(evar, ':');
if (3 != pmix_argv_count(uri)) {
pmix_argv_free(uri);
return PMIX_ERROR;
}
/* set the server nspace */
pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t);
pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t);
(void)strncpy(pmix_client_globals.myserver.info->nptr->nspace, uri[0], PMIX_MAX_NSLEN);
/* set the server rank */
pmix_client_globals.myserver.info->rank = strtoull(uri[1], NULL, 10);
/* setup the path to the daemon rendezvous point */
memset(&address, 0, sizeof(struct sockaddr_un));
address.sun_family = AF_UNIX;
snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s", uri[2]);
/* if the rendezvous file doesn't exist, that's an error */
if (0 != access(uri[2], R_OK)) {
pmix_argv_free(uri);
return PMIX_ERR_NOT_FOUND;
}
pmix_argv_free(uri);
/* we also require our rank */
if (NULL == (evar = getenv("PMIX_RANK"))) {
/* let the caller know that the server isn't available yet */
@ -348,21 +275,45 @@ PMIX_EXPORT pmix_status_t PMIx_Init(pmix_proc_t *proc,
}
pmix_globals.pindex = -1;
/* setup the support */
/* select our psec compat module - the selection will be based
* on the corresponding envars that should have been passed
* to us at launch */
evar = getenv("PMIX_SECURITY_MODE");
if (PMIX_SUCCESS != (rc = pmix_psec.assign_module(pmix_globals.mypeer, evar))) {
return PMIX_ERR_INIT;
}
/* the server will be using the same */
pmix_client_globals.myserver.compat.psec = pmix_globals.mypeer->compat.psec;
/* setup the shared memory support */
#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
if (PMIX_SUCCESS != (rc = pmix_dstore_init(NULL, 0))) {
return PMIX_ERR_DATA_VALUE_NOT_FOUND;
}
#endif /* PMIX_ENABLE_DSTORE */
/* setup an object to track server connection */
/* connect to the server */
if (PMIX_SUCCESS != (rc = pmix_ptl.connect_to_peer(&pmix_client_globals.myserver, info, ninfo))){
return rc;
}
/* send a request for our job info - we do this as a non-blocking
* transaction because some systems cannot handle very large
* blocking operations and error out if we try them. */
req = PMIX_NEW(pmix_buffer_t);
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(req, &cmd, 1, PMIX_CMD))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(req);
return rc;
}
/* send to the server */
PMIX_CONSTRUCT(&cb, pmix_cb_t);
cb.active = true;
/* connect to the server - returns job info if successful */
if (PMIX_SUCCESS != (rc = connect_to_server(&address, &cb))){
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, req, job_data, (void*)&cb))){
PMIX_DESTRUCT(&cb);
return rc;
}
/* wait for the data to return */
PMIX_WAIT_FOR_COMPLETION(cb.active);
rc = cb.status;
PMIX_DESTRUCT(&cb);
@ -384,10 +335,10 @@ PMIX_EXPORT int PMIx_Initialized(void)
PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
{
pmix_buffer_t *msg;
pmix_cb_t *cb;
pmix_cmd_t cmd = PMIX_FINALIZE_CMD;
pmix_status_t rc;
size_t n;
volatile bool active;
if (1 != pmix_globals.init_cntr) {
--pmix_globals.init_cntr;
@ -430,21 +381,19 @@ PMIX_EXPORT pmix_status_t PMIx_Finalize(const pmix_info_t info[], size_t ninfo)
return rc;
}
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = PMIX_NEW(pmix_cb_t);
cb->active = true;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:client sending finalize sync to server");
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
/* send to the server */
active = true;;
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg,
wait_cbfunc, (void*)&active))){
return rc;
}
/* wait for the ack to return */
PMIX_WAIT_FOR_COMPLETION(cb->active);
PMIX_RELEASE(cb);
PMIX_WAIT_FOR_COMPLETION(active);
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:client finalize sync received");
}
@ -471,7 +420,7 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int flag, const char msg[],
pmix_buffer_t *bfr;
pmix_cmd_t cmd = PMIX_ABORT_CMD;
pmix_status_t rc;
pmix_cb_t *cb;
pmix_ptl_sr_t cb;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:client abort called");
@ -523,15 +472,20 @@ PMIX_EXPORT pmix_status_t PMIx_Abort(int flag, const char msg[],
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
cb = PMIX_NEW(pmix_cb_t);
cb->active = true;
PMIX_CONSTRUCT(&cb, pmix_ptl_sr_t);
cb.active = true;
cb.cbfunc = wait_cbfunc;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, bfr, wait_cbfunc, cb);
/* send to the server */
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, bfr,
wait_cbfunc, &cb))){
PMIX_DESTRUCT(&cb);
return rc;
}
/* wait for the release */
PMIX_WAIT_FOR_COMPLETION(cb->active);
PMIX_RELEASE(cb);
PMIX_WAIT_FOR_COMPLETION(cb.active);
PMIX_DESTRUCT(&cb);
return PMIX_SUCCESS;
}
@ -672,14 +626,17 @@ static void _commitfn(int sd, short args, void *cbdata)
PMIX_RELEASE(pmix_globals.cache_remote);
}
/* push the message into our event base to send to the server - always
* send, even if we have nothing to contribute, so the server knows
/* always send, even if we have nothing to contribute, so the server knows
* that we contributed whatever we had */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msgout, NULL, NULL);
if (PMIX_SUCCESS == (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msgout,
wait_cbfunc, (void*)&cb->active))){
cb->pstatus = PMIX_SUCCESS;
return;
}
done:
cb->pstatus = rc;
cb->active = false;
done:
cb->pstatus = rc;
cb->active = false;
}
PMIX_EXPORT pmix_status_t PMIx_Commit(void)
@ -874,247 +831,3 @@ PMIX_EXPORT pmix_status_t PMIx_Resolve_nodes(const char *nspace, char **nodelist
return rc;
}
static pmix_status_t send_connect_ack(int sd)
{
char *msg;
pmix_usock_hdr_t hdr;
size_t sdsize=0, csize=0;
char *cred = NULL;
char *sec;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: SEND CONNECT ACK");
/* setup the header */
memset(&hdr, 0, sizeof(pmix_usock_hdr_t));
hdr.pindex = -1;
hdr.tag = UINT32_MAX;
/* reserve space for the nspace and rank info */
sdsize = strlen(pmix_globals.myid.nspace) + 1 + sizeof(int);
/* a security module was assigned to us during rte_init based
* on a list of available security modules provided by our
* local PMIx server. Now use that module to
* get a credential, if the security system provides one. Not
* every psec module will do so, thus we must first check */
if (NULL != pmix_globals.mypeer->compat.psec->create_cred) {
if (NULL == (cred = pmix_globals.mypeer->compat.psec->create_cred())) {
/* an error occurred - we cannot continue */
return PMIX_ERR_INVALID_CRED;
}
csize = strlen(cred) + 1; // must NULL terminate the string!
}
/* add our active bfrops and sec module info, and what type
* of buffers we are using */
// bfrop = pmix_globals.mypeer->compat.bfrops->name;
sec = pmix_globals.mypeer->compat.psec->name;
/* set the number of bytes to be read beyond the header */
hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 + csize; // must NULL terminate the VERSION string!
/* create a space for our message */
sdsize = (sizeof(hdr) + hdr.nbytes);
if (NULL == (msg = (char*)malloc(sdsize))) {
if (NULL != cred) {
free(cred);
}
return PMIX_ERR_OUT_OF_RESOURCE;
}
memset(msg, 0, sdsize);
/* load the message */
csize=0;
memcpy(msg, &hdr, sizeof(pmix_usock_hdr_t));
csize += sizeof(pmix_usock_hdr_t);
memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace));
csize += strlen(pmix_globals.myid.nspace)+1;
memcpy(msg+csize, &pmix_globals.myid.rank, sizeof(int));
csize += sizeof(int);
memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
csize += strlen(PMIX_VERSION)+1;
// memcpy(msg+csize, bfrop, strlen(bfrop));
// csize += strlen(bfrop)+1;
memcpy(msg+csize, sec, strlen(sec));
csize += strlen(sec)+1;
// memcpy(msg+csize, &pmix_globals.mypeer->comm.type, sizeof(pmix_bfrop_buffer_type_t));
// csize += sizeof(pmix_bfrop_buffer_type_t);
if (NULL != cred) {
memcpy(msg+csize, cred, strlen(cred)); // leaves last position in msg set to NULL
}
if (PMIX_SUCCESS != pmix_usock_send_blocking(sd, msg, sdsize)) {
free(msg);
if (NULL != cred) {
free(cred);
}
return PMIX_ERR_UNREACH;
}
free(msg);
if (NULL != cred) {
free(cred);
}
return PMIX_SUCCESS;
}
/* we receive a connection acknowledgement from the server,
* consisting of nothing more than a status report. If success,
* then we initiate authentication method */
static pmix_status_t recv_connect_ack(int sd)
{
pmix_status_t reply;
pmix_status_t rc;
struct timeval tv, save;
pmix_socklen_t sz;
bool sockopt = true;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT ACK FROM SERVER");
/* get the current timeout value so we can reset to it */
sz = sizeof(save);
if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) {
if (ENOPROTOOPT == errno) {
sockopt = false;
} else {
return PMIX_ERR_UNREACH;
}
} else {
/* set a timeout on the blocking recv so we don't hang */
tv.tv_sec = 2;
tv.tv_usec = 0;
if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO");
return PMIX_ERR_UNREACH;
}
}
/* receive the status reply */
rc = pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(int));
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
return rc;
}
/* see if they want us to do the handshake */
if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
if (NULL == pmix_globals.mypeer->compat.psec->client_handshake) {
return PMIX_ERR_HANDSHAKE_FAILED;
}
if (PMIX_SUCCESS != (rc = pmix_globals.mypeer->compat.psec->client_handshake(sd))) {
return rc;
}
} else if (PMIX_SUCCESS != reply) {
return reply;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT CONFIRMATION");
/* receive our index into the server's client array */
rc = pmix_usock_recv_blocking(sd, (char*)&pmix_globals.pindex, sizeof(int));
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
return rc;
}
if (sockopt) {
/* return the socket to normal */
if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
return PMIX_ERR_UNREACH;
}
}
return PMIX_SUCCESS;
}
static pmix_status_t usock_connect(struct sockaddr *addr, int *fd)
{
int sd=-1;
pmix_status_t rc;
pmix_socklen_t addrlen = 0;
int retries = 0;
pmix_output_verbose(2, pmix_globals.debug_output,
"usock_peer_try_connect: attempting to connect to server");
addrlen = sizeof(struct sockaddr_un);
while (retries < PMIX_MAX_RETRIES) {
retries++;
/* Create the new socket */
sd = socket(PF_UNIX, SOCK_STREAM, 0);
if (sd < 0) {
pmix_output(0, "pmix:create_socket: socket() failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
continue;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"usock_peer_try_connect: attempting to connect to server on socket %d", sd);
/* try to connect */
if (connect(sd, addr, addrlen) < 0) {
if (pmix_socket_errno == ETIMEDOUT) {
/* The server may be too busy to accept new connections */
pmix_output_verbose(2, pmix_globals.debug_output,
"timeout connecting to server");
CLOSE_THE_SOCKET(sd);
continue;
}
/* Some kernels (Linux 2.6) will automatically software
abort a connection that was ECONNREFUSED on the last
attempt, without even trying to establish the
connection. Handle that case in a semi-rational
way by trying twice before giving up */
if (ECONNABORTED == pmix_socket_errno) {
pmix_output_verbose(2, pmix_globals.debug_output,
"connection to server aborted by OS - retrying");
CLOSE_THE_SOCKET(sd);
continue;
} else {
pmix_output_verbose(2, pmix_globals.debug_output,
"Connect failed: %s (%d)", strerror(pmix_socket_errno),
pmix_socket_errno);
CLOSE_THE_SOCKET(sd);
continue;
}
} else {
/* otherwise, the connect succeeded - so break out of the loop */
break;
}
}
if (retries == PMIX_MAX_RETRIES || sd < 0){
/* We were unsuccessful in establishing this connection, and are
* not likely to suddenly become successful */
if (0 <= sd) {
CLOSE_THE_SOCKET(sd);
}
return PMIX_ERR_UNREACH;
}
/* send our identity and any authentication credentials to the server */
if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) {
CLOSE_THE_SOCKET(sd);
return rc;
}
/* do whatever handshake is required */
if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) {
CLOSE_THE_SOCKET(sd);
return rc;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"sock_peer_try_connect: Connection across to server succeeded");
/* mark the connection as made */
pmix_globals.connected = true;
pmix_usock_set_nonblocking(sd);
*fd = sd;
return PMIX_SUCCESS;
}

22
opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_connect.c
Просмотреть файл

@ -51,13 +51,14 @@
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/output.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#include "pmix_client_ops.h"
#include "src/include/pmix_jobdata.h"
/* callback for wait completion */
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
static void op_cbfunc(pmix_status_t status, void *cbdata);
@ -167,9 +168,12 @@ PMIX_EXPORT pmix_status_t PMIx_Connect_nb(const pmix_proc_t procs[], size_t npro
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){
PMIX_RELEASE(msg);
PMIX_RELEASE(cb);
}
return PMIX_SUCCESS;
return rc;
}
PMIX_EXPORT pmix_status_t PMIx_Disconnect(const pmix_proc_t procs[], size_t nprocs,
@ -274,15 +278,19 @@ PMIX_EXPORT pmix_status_t PMIx_Disconnect_nb(const pmix_proc_t procs[], size_t n
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){
PMIX_RELEASE(msg);
PMIX_RELEASE(cb);
}
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: disconnect completed");
return PMIX_SUCCESS;
return rc;
}
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_cb_t *cb = (pmix_cb_t*)cbdata;

14
opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_fence.c
Просмотреть файл

@ -52,7 +52,7 @@
#include "src/util/error.h"
#include "src/util/hash.h"
#include "src/util/output.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#include "pmix_client_ops.h"
@ -61,7 +61,7 @@ static pmix_status_t pack_fence(pmix_buffer_t *msg, pmix_cmd_t cmd,
const pmix_proc_t *procs, size_t nprocs,
const pmix_info_t *info, size_t ninfo);
static void wait_cbfunc(struct pmix_peer_t *pr,
pmix_usock_hdr_t *hdr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
static void op_cbfunc(pmix_status_t status, void *cbdata);
@ -160,9 +160,11 @@ PMIX_EXPORT pmix_status_t PMIx_Fence_nb(const pmix_proc_t procs[], size_t nprocs
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
return PMIX_SUCCESS;
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){
PMIX_RELEASE(msg);
PMIX_RELEASE(cb);
}
return rc;
}
static pmix_status_t unpack_return(pmix_buffer_t *data)
@ -223,7 +225,7 @@ static pmix_status_t pack_fence(pmix_buffer_t *msg, pmix_cmd_t cmd,
return PMIX_SUCCESS;
}
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_cb_t *cb = (pmix_cb_t*)cbdata;

23
opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c
Просмотреть файл

@ -53,7 +53,7 @@
#include "src/util/error.h"
#include "src/util/hash.h"
#include "src/util/output.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
#include "src/dstore/pmix_dstore.h"
#endif /* PMIX_ENABLE_DSTORE */
@ -67,8 +67,9 @@ static pmix_buffer_t* _pack_get(char *nspace, pmix_rank_t rank,
static void _getnbfn(int sd, short args, void *cbdata);
static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
static void _getnb_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
static void _value_cbfunc(pmix_status_t status, pmix_value_t *kv, void *cbdata);
@ -234,7 +235,8 @@ static pmix_buffer_t* _pack_get(char *nspace, pmix_rank_t rank,
/* this callback is coming from the usock recv, and thus
* is occurring inside of our progress thread - hence, no
* need to thread shift */
static void _getnb_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void _getnb_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_cb_t *cb = (pmix_cb_t*)cbdata;
@ -737,10 +739,13 @@ static void _getnbfn(int fd, short flags, void *cbdata)
pmix_globals.myid.nspace, pmix_globals.myid.rank,
cb->nspace, cb->rank, cb->key);
/* create a callback object as we need to pass it to the
* recv routine so we know which callback to use when
* the return message is recvd */
/* track the callback object */
pmix_list_append(&pmix_client_globals.pending_requests, &cb->super);
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, _getnb_cbfunc, cb);
/* send to the server */
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, _getnb_cbfunc, (void*)cb))){
pmix_list_remove_item(&pmix_client_globals.pending_requests, &cb->super);
cb->value_cbfunc(PMIX_ERROR, NULL, cb->cbdata);
PMIX_RELEASE(cb);
return;
}
}

1
opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_ops.h
Просмотреть файл

@ -15,7 +15,6 @@
#include "src/buffer_ops/buffer_ops.h"
#include "src/class/pmix_hash_table.h"
#include "src/usock/usock.h"
BEGIN_C_DECLS

41
opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_pub.c
Просмотреть файл

@ -51,14 +51,16 @@
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/output.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#include "pmix_client_ops.h"
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
static void op_cbfunc(pmix_status_t status, void *cbdata);
static void wait_lookup_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_lookup_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
static void lookup_cbfunc(pmix_status_t status, pmix_pdata_t pdata[], size_t ndata,
void *cbdata);
@ -163,13 +165,16 @@ PMIX_EXPORT pmix_status_t PMIx_Publish_nb(const pmix_info_t info[], size_t ninfo
cb->active = true;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){
PMIX_RELEASE(msg);
PMIX_RELEASE(cb);
}
return PMIX_SUCCESS;
return rc;
}
PMIX_EXPORT pmix_status_t PMIx_Lookup(pmix_pdata_t pdata[], size_t ndata,
const pmix_info_t info[], size_t ninfo)
const pmix_info_t info[], size_t ninfo)
{
pmix_status_t rc;
pmix_cb_t *cb;
@ -289,10 +294,13 @@ PMIX_EXPORT pmix_status_t PMIx_Lookup_nb(char **keys,
cb->lookup_cbfunc = cbfunc;
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_lookup_cbfunc, cb);
/* send to the server */
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_lookup_cbfunc, (void*)cb))){
PMIX_RELEASE(msg);
PMIX_RELEASE(cb);
}
return PMIX_SUCCESS;
return rc;
}
PMIX_EXPORT pmix_status_t PMIx_Unpublish(char **keys,
@ -391,13 +399,17 @@ PMIX_EXPORT pmix_status_t PMIx_Unpublish_nb(char **keys,
cb->cbdata = cbdata;
cb->active = true;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
/* send to the server */
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){
PMIX_RELEASE(msg);
PMIX_RELEASE(cb);
}
return PMIX_SUCCESS;
return rc;
}
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_cb_t *cb = (pmix_cb_t*)cbdata;
@ -428,7 +440,8 @@ static void op_cbfunc(pmix_status_t status, void *cbdata)
cb->active = false;
}
static void wait_lookup_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_lookup_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_cb_t *cb = (pmix_cb_t*)cbdata;

15
opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_spawn.c
Просмотреть файл

@ -51,12 +51,13 @@
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/output.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#include "pmix_client_ops.h"
#include "src/include/pmix_jobdata.h"
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
static void spawn_cbfunc(pmix_status_t status, char nspace[], void *cbdata);
@ -169,13 +170,17 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, wait_cbfunc, cb);
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, wait_cbfunc, (void*)cb))){
PMIX_RELEASE(msg);
PMIX_RELEASE(cb);
}
return PMIX_SUCCESS;
return rc;
}
/* callback for wait completion */
static void wait_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void wait_cbfunc(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_cb_t *cb = (pmix_cb_t*)cbdata;

11
opal/mca/pmix/pmix2x/pmix/src/common/pmix_log.c
Просмотреть файл

@ -25,14 +25,14 @@
#include "src/util/error.h"
#include "src/util/output.h"
#include "src/buffer_ops/buffer_ops.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#include "src/client/pmix_client_ops.h"
#include "src/server/pmix_server_ops.h"
#include "src/include/pmix_globals.h"
static void log_cbfunc(struct pmix_peer_t *peer,
pmix_usock_hdr_t *hdr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata;
@ -84,6 +84,7 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
pmix_host_server.log(&pmix_globals.myid,
data, ndata, directives, ndirs,
cbfunc, cbdata);
rc = PMIX_SUCCESS;
} else {
/* if we are a client, then relay this request to the server */
cd = PMIX_NEW(pmix_shift_caddy_t);
@ -125,7 +126,9 @@ PMIX_EXPORT pmix_status_t PMIx_Log_nb(const pmix_info_t data[], size_t ndata,
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:query sending to server");
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, log_cbfunc, cd);
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, log_cbfunc, (void*)cd))){
PMIX_RELEASE(cd);
}
}
return PMIX_SUCCESS;
return rc;
}

11
opal/mca/pmix/pmix2x/pmix/src/common/pmix_query.c
Просмотреть файл

@ -25,7 +25,7 @@
#include "src/util/error.h"
#include "src/util/output.h"
#include "src/buffer_ops/buffer_ops.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#include "src/client/pmix_client_ops.h"
#include "src/server/pmix_server_ops.h"
@ -44,7 +44,7 @@ static void relcbfunc(void *cbdata)
PMIX_RELEASE(cd);
}
static void query_cbfunc(struct pmix_peer_t *peer,
pmix_usock_hdr_t *hdr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
@ -124,6 +124,7 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque
pmix_host_server.query(&pmix_globals.myid,
queries, nqueries,
cbfunc, cbdata);
rc = PMIX_SUCCESS;
} else {
/* if we are a client, then relay this request to the server */
cd = PMIX_NEW(pmix_query_caddy_t);
@ -150,7 +151,9 @@ PMIX_EXPORT pmix_status_t PMIx_Query_info_nb(pmix_query_t queries[], size_t nque
}
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:query sending to server");
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, query_cbfunc, cd);
if (PMIX_SUCCESS != (rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, query_cbfunc, (void*)cd))){
PMIX_RELEASE(cd);
}
}
return PMIX_SUCCESS;
return rc;
}

11
opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_notification.c
Просмотреть файл

@ -62,7 +62,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status,
return rc;
}
static void notify_event_cbfunc(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void notify_event_cbfunc(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_status_t rc, ret;
@ -155,8 +155,13 @@ static pmix_status_t notify_server_of_event(pmix_status_t status,
cb = PMIX_NEW(pmix_cb_t);
cb->op_cbfunc = cbfunc;
cb->cbdata = cbdata;
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, notify_event_cbfunc, cb);
/* send to the server */
rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, notify_event_cbfunc, cb);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(cb);
goto cleanup;
}
/* now notify any matching registered callbacks we have */
pmix_invoke_local_event_hdlr(chain);

17
opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c
Просмотреть файл

@ -59,7 +59,7 @@ PMIX_CLASS_INSTANCE(pmix_rshift_caddy_t,
rscon, rsdes);
static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr,
static void regevents_cbfunc(struct pmix_peer_t *peer, pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_rshift_caddy_t *rb = (pmix_rshift_caddy_t*)cbdata;
@ -152,9 +152,13 @@ static pmix_status_t _send_to_server(pmix_rshift_caddy_t *rcd)
return rc;
}
}
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, regevents_cbfunc, rcd);
rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, regevents_cbfunc, rcd);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(msg);
}
return PMIX_SUCCESS;
return rc;
}
static pmix_status_t _add_hdlr(pmix_list_t *list, pmix_list_item_t *item,
@ -565,8 +569,11 @@ static void dereg_event_hdlr(int sd, short args, void *cbdata)
report:
if (NULL != msg) {
/* push the message into our event base to send to the server */
PMIX_ACTIVATE_SEND_RECV(&pmix_client_globals.myserver, msg, NULL, NULL);
/* send to the server */
rc = pmix_ptl.send_recv(&pmix_client_globals.myserver, msg, NULL, NULL);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
}
}
cleanup:

79
opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.c
Просмотреть файл

@ -40,26 +40,75 @@
#include "src/class/pmix_hash_table.h"
#include "src/class/pmix_list.h"
void pmix_globals_init(void)
static void cbcon(pmix_cb_t *p)
{
memset(&pmix_globals.myid, 0, sizeof(pmix_proc_t));
PMIX_CONSTRUCT(&pmix_globals.nspaces, pmix_list_t);
PMIX_CONSTRUCT(&pmix_globals.events, pmix_events_t);
p->active = false;
p->checked = false;
PMIX_CONSTRUCT(&p->data, pmix_buffer_t);
p->cbfunc = NULL;
p->op_cbfunc = NULL;
p->value_cbfunc = NULL;
p->lookup_cbfunc = NULL;
p->spawn_cbfunc = NULL;
p->cbdata = NULL;
memset(p->nspace, 0, PMIX_MAX_NSLEN+1);
p->rank = -1;
p->key = NULL;
p->value = NULL;
p->procs = NULL;
p->info = NULL;
p->ninfo = 0;
p->nvals = 0;
}
void pmix_globals_finalize(void)
static void cbdes(pmix_cb_t *p)
{
PMIX_LIST_DESTRUCT(&pmix_globals.nspaces);
if (NULL != pmix_globals.cache_local) {
PMIX_RELEASE(pmix_globals.cache_local);
}
if (NULL != pmix_globals.cache_remote) {
PMIX_RELEASE(pmix_globals.cache_remote);
}
PMIX_DESTRUCT(&pmix_globals.events);
PMIX_DESTRUCT(&p->data);
}
PMIX_CLASS_INSTANCE(pmix_cb_t,
pmix_list_item_t,
cbcon, cbdes);
static void pcon(pmix_peer_t *p)
{
p->info = NULL;
p->proc_cnt = 0;
p->server_object = NULL;
p->index = 0;
p->sd = -1;
p->send_ev_active = false;
p->recv_ev_active = false;
PMIX_CONSTRUCT(&p->send_queue, pmix_list_t);
p->send_msg = NULL;
p->recv_msg = NULL;
memset(&p->compat, 0, sizeof(p->compat));
}
static void pdes(pmix_peer_t *p)
{
if (0 <= p->sd) {
CLOSE_THE_SOCKET(p->sd);
}
if (p->send_ev_active) {
event_del(&p->send_event);
}
if (p->recv_ev_active) {
event_del(&p->recv_event);
}
if (NULL != p->info) {
PMIX_RELEASE(p->info);
}
PMIX_LIST_DESTRUCT(&p->send_queue);
if (NULL != p->send_msg) {
PMIX_RELEASE(p->send_msg);
}
if (NULL != p->recv_msg) {
PMIX_RELEASE(p->recv_msg);
}
}
PMIX_CLASS_INSTANCE(pmix_peer_t,
pmix_object_t,
pcon, pdes);
static void nscon(pmix_nspace_t *p)
{

91
opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h
Просмотреть файл

@ -38,6 +38,7 @@
#include "src/event/pmix_event.h"
#include "src/mca/psec/psec.h"
#include "src/mca/ptl/ptl.h"
BEGIN_C_DECLS
@ -92,14 +93,11 @@ typedef enum {
PMIX_PROC_TOOL
} pmix_proc_type_t;
/* defins some convenience macros for testing proc type */
#define PMIX_PROC_IS_CLIENT (PMIX_PROC_CLIENT == pmix_globals.proc_type)
#define PMIX_PROC_IS_SERVER (PMIX_PROC_SERVER == pmix_globals.proc_type)
#define PMIX_PROC_IS_TOOL (PMIX_PROC_TOOL == pmix_globals.proc_type)
/**** MESSAGING STRUCTURES ****/
/* header for messages */
typedef struct {
int pindex;
uint32_t tag;
size_t nbytes;
} pmix_usock_hdr_t;
/* internally used object for transferring data
* to/from the server and for storing in the
@ -114,37 +112,6 @@ PMIX_CLASS_DECLARATION(pmix_kval_t);
// forward declaration
struct pmix_peer_t;
/* internally used cbfunc */
typedef void (*pmix_usock_cbfunc_t)(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
/* usock structure for sending a message */
typedef struct {
pmix_list_item_t super;
pmix_event_t ev;
pmix_usock_hdr_t hdr;
pmix_buffer_t *data;
bool hdr_sent;
char *sdptr;
size_t sdbytes;
} pmix_usock_send_t;
PMIX_CLASS_DECLARATION(pmix_usock_send_t);
/* usock structure for recving a message */
typedef struct {
pmix_list_item_t super;
pmix_event_t ev;
struct pmix_peer_t *peer;
int sd;
pmix_usock_hdr_t hdr;
char *data;
bool hdr_recvd;
char *rdptr;
size_t rdbytes;
} pmix_usock_recv_t;
PMIX_CLASS_DECLARATION(pmix_usock_recv_t);
/**** PEER STRUCTURES ****/
/* objects for tracking active nspaces */
typedef struct {
@ -185,6 +152,7 @@ PMIX_CLASS_DECLARATION(pmix_rank_info_t);
* to plugins for cross-version support */
typedef struct pmix_personality_t {
pmix_psec_module_t *psec;
pmix_ptl_module_t *ptl;
} pmix_personality_t;
/* object for tracking peers - each peer can have multiple
@ -199,13 +167,13 @@ typedef struct pmix_peer_t {
void *server_object;
int index;
int sd;
pmix_event_t send_event; /**< registration with event thread for send events */
pmix_event_t send_event; /**< registration with event thread for send events */
bool send_ev_active;
pmix_event_t recv_event; /**< registration with event thread for recv events */
pmix_event_t recv_event; /**< registration with event thread for recv events */
bool recv_ev_active;
pmix_list_t send_queue; /**< list of messages to send */
pmix_usock_send_t *send_msg; /**< current send in progress */
pmix_usock_recv_t *recv_msg; /**< current recv in progress */
pmix_list_t send_queue; /**< list of messages to send */
pmix_ptl_send_t *send_msg; /**< current send in progress */
pmix_ptl_recv_t *recv_msg; /**< current recv in progress */
pmix_personality_t compat;
} pmix_peer_t;
PMIX_CLASS_DECLARATION(pmix_peer_t);
@ -230,7 +198,7 @@ PMIX_CLASS_DECLARATION(pmix_snd_caddy_t);
* request into the server's event base */
typedef struct {
pmix_list_item_t super;
pmix_usock_hdr_t hdr;
pmix_ptl_hdr_t hdr;
pmix_peer_t *peer;
pmix_snd_caddy_t snd;
} pmix_server_caddy_t;
@ -323,6 +291,35 @@ PMIX_CLASS_DECLARATION(pmix_job_data_caddy_t);
} pmix_shift_caddy_t;
PMIX_CLASS_DECLARATION(pmix_shift_caddy_t);
/* struct for tracking ops */
typedef struct {
pmix_list_item_t super;
pmix_event_t ev;
volatile bool active;
bool checked;
int status;
pmix_status_t pstatus;
pmix_scope_t scope;
pmix_buffer_t data;
pmix_ptl_cbfunc_t cbfunc;
pmix_op_cbfunc_t op_cbfunc;
pmix_value_cbfunc_t value_cbfunc;
pmix_lookup_cbfunc_t lookup_cbfunc;
pmix_spawn_cbfunc_t spawn_cbfunc;
pmix_evhdlr_reg_cbfunc_t errreg_cbfunc;
size_t errhandler_ref;
void *cbdata;
char nspace[PMIX_MAX_NSLEN+1];
pmix_rank_t rank;
char *key;
pmix_value_t *value;
pmix_proc_t *procs;
pmix_info_t *info;
size_t ninfo;
size_t nvals;
} pmix_cb_t;
PMIX_CLASS_DECLARATION(pmix_cb_t);
/* define a very simple caddy for dealing with pmix_info_t
* objects when transferring portions of arrays */
typedef struct {
@ -371,12 +368,6 @@ typedef struct {
} pmix_globals_t;
/* initialize the pmix_global structure */
void pmix_globals_init(void);
/* finalize the pmix_global structure */
void pmix_globals_finalize(void);
extern pmix_globals_t pmix_globals;
END_C_DECLS

4
opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c
Просмотреть файл

@ -1926,13 +1926,13 @@ static char *source_name(pmix_mca_base_var_t *var)
static int var_value_string (pmix_mca_base_var_t *var, char **value_string)
{
const pmix_mca_base_var_storage_t *value;
const pmix_mca_base_var_storage_t *value=NULL;
int ret;
assert (PMIX_MCA_BASE_VAR_TYPE_MAX > var->mbv_type);
ret = pmix_mca_base_var_get_value(var->mbv_index, &value, NULL, NULL);
if (PMIX_SUCCESS !=ret) {
if (PMIX_SUCCESS != ret || NULL == value) {
return ret;
}

2
opal/mca/pmix/pmix2x/pmix/src/mca/pif/pif.h
Просмотреть файл

@ -89,7 +89,7 @@ BEGIN_C_DECLS
typedef struct pmix_pif_t {
pmix_list_item_t super;
char if_name[IF_NAMESIZE];
char if_name[IF_NAMESIZE+1];
int if_index;
uint16_t if_kernel_index;
uint16_t af_family;

12
opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/base.h
Просмотреть файл

@ -80,6 +80,18 @@ typedef struct pmix_psec_globals_t pmix_psec_globals_t;
extern pmix_psec_globals_t pmix_psec_globals;
PMIX_EXPORT char* pmix_psec_base_get_available_modules(void);
PMIX_EXPORT pmix_status_t pmix_psec_base_assign_module(struct pmix_peer_t *peer,
const char *options);
PMIX_EXPORT pmix_status_t pmix_psec_base_create_cred(struct pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char **cred, size_t *len);
PMIX_EXPORT pmix_status_t pmix_psec_base_client_handshake(struct pmix_peer_t *peer, int sd);
PMIX_EXPORT pmix_status_t pmix_psec_base_validate_connection(struct pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char *cred, size_t len);
END_C_DECLS
#endif

84
opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_fns.c
Просмотреть файл

@ -18,8 +18,9 @@
#include "src/class/pmix_list.h"
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/output.h"
#include "src/mca/ptl/base/base.h"
#include "src/mca/psec/base/base.h"
@ -43,16 +44,19 @@ char* pmix_psec_base_get_available_modules(void)
return reply;
}
pmix_psec_module_t* pmix_psec_base_assign_module(const char *options)
pmix_status_t pmix_psec_base_assign_module(struct pmix_peer_t *peer,
const char *options)
{
pmix_peer_t *pr = (pmix_peer_t*)peer;
pmix_psec_base_active_module_t *active;
pmix_psec_module_t *mod;
char **tmp=NULL;
int i;
pmix_psec_module_t *mod;
if (!pmix_psec_globals.initialized) {
return NULL;
return PMIX_ERR_INIT;
}
if (NULL != options) {
tmp = pmix_argv_split(options, ',');
}
@ -60,14 +64,16 @@ pmix_psec_module_t* pmix_psec_base_assign_module(const char *options)
PMIX_LIST_FOREACH(active, &pmix_psec_globals.actives, pmix_psec_base_active_module_t) {
if (NULL == tmp) {
if (NULL != (mod = active->component->assign_module())) {
return mod;
pr->compat.psec = mod;
return PMIX_SUCCESS;
}
} else {
for (i=0; NULL != tmp[i]; i++) {
if (0 == strcmp(tmp[i], active->component->base.pmix_mca_component_name)) {
if (NULL != (mod = active->component->assign_module())) {
pmix_argv_free(tmp);
return mod;
pr->compat.psec = mod;
return PMIX_SUCCESS;
}
}
}
@ -78,5 +84,69 @@ pmix_psec_module_t* pmix_psec_base_assign_module(const char *options)
if (NULL != tmp) {
pmix_argv_free(tmp);
}
return NULL;
return PMIX_ERR_NOT_AVAILABLE;
}
pmix_status_t pmix_psec_base_create_cred(struct pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char **cred, size_t *len)
{
pmix_peer_t *pr = (pmix_peer_t*)peer;
if (NULL == pr->compat.psec->create_cred) {
return PMIX_ERR_NOT_SUPPORTED;
}
return pr->compat.psec->create_cred(protocol, cred, len);
}
pmix_status_t pmix_psec_base_client_handshake(struct pmix_peer_t *peer, int sd)
{
pmix_peer_t *pr = (pmix_peer_t*)peer;
if (NULL == pr->compat.psec->client_handshake) {
return PMIX_ERR_NOT_SUPPORTED;
}
return pr->compat.psec->client_handshake(sd);
}
pmix_status_t pmix_psec_base_validate_connection(struct pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char *cred, size_t len)
{
pmix_peer_t *pr = (pmix_peer_t*)peer;
pmix_status_t rc;
/* if a credential is available, then check it */
if (NULL != pr->compat.psec->validate_cred) {
if (PMIX_SUCCESS != (rc = pr->compat.psec->validate_cred(peer, protocol, cred, len))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"validation of credential failed: %s",
PMIx_Error_string(rc));
return rc;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"credential validated");
/* send them success */
rc = PMIX_SUCCESS;
if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pr->sd, (char*)&rc, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
}
return rc;
} else if (NULL != pr->compat.psec->server_handshake) {
/* execute the handshake if the security mode calls for it */
pmix_output_verbose(2, pmix_globals.debug_output,
"executing handshake");
rc = PMIX_ERR_READY_FOR_HANDSHAKE;
if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pr->sd, (char*)&rc, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
return rc;
}
if (PMIX_SUCCESS != (rc = pr->compat.psec->server_handshake(peer))) {
PMIX_ERROR_LOG(rc);
}
return rc;
} else {
/* this is not allowed */
return PMIX_ERR_NOT_SUPPORTED;
}
}

7
opal/mca/pmix/pmix2x/pmix/src/mca/psec/base/psec_base_frame.c
Просмотреть файл

@ -45,6 +45,13 @@
/* Instantiate the global vars */
pmix_psec_globals_t pmix_psec_globals = {{{0}}};
pmix_psec_API_t pmix_psec = {
.get_available_modules = pmix_psec_base_get_available_modules,
.assign_module = pmix_psec_base_assign_module,
.create_cred = pmix_psec_base_create_cred,
.client_handshake = pmix_psec_base_client_handshake,
.validate_connection = pmix_psec_base_validate_connection
};
static pmix_status_t pmix_psec_close(void)
{

25
opal/mca/pmix/pmix2x/pmix/src/mca/psec/munge/psec_munge.c
Просмотреть файл

@ -28,10 +28,12 @@
#include "src/mca/psec/psec.h"
#include "psec_munge.h"
static int munge_init(void);
static pmix_status_t munge_init(void);
static void munge_finalize(void);
static char* create_cred(void);
static int validate_cred(pmix_peer_t *peer, char *cred);
static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
char **cred, size_t *len);
static pmix_status_t validate_cred(pmix_listener_protocol_t protocol,
pmix_peer_t *peer, char *cred, size_t len);
pmix_psec_module_t pmix_munge_module = {
"munge",
@ -47,7 +49,7 @@ static char *mycred = NULL;
static bool initialized = false;
static bool refresh = false;
static int munge_init(void)
static pmix_status_t munge_init(void)
{
int rc;
@ -81,10 +83,10 @@ static void munge_finalize(void)
}
}
static char* create_cred(void)
static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
char **cred, size_t *len)
{
int rc;
char *resp=NULL;
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: munge create_cred");
@ -92,7 +94,8 @@ static char* create_cred(void)
if (initialized) {
if (!refresh) {
refresh = true;
resp = strdup(mycred);
*cred = strdup(mycred);
*len = strlen(mycred) + 1;
} else {
/* munge does not allow reuse of a credential, so we have to
* refresh it for every use */
@ -105,13 +108,15 @@ static char* create_cred(void)
munge_strerror(rc));
return NULL;
}
resp = strdup(mycred);
*cred = strdup(mycred);
*len = strlen(mycred) + 1;
}
}
return resp;
return PMIX_SUCCESS;
}
static int validate_cred(pmix_peer_t *peer, char *cred)
static pmix_status_t validate_cred(pmix_listener_protocol_t protocol,
pmix_peer_t *peer, char *cred, size_t len)
{
uid_t uid;
gid_t gid;

174
opal/mca/pmix/pmix2x/pmix/src/mca/psec/native/psec_native.c
Просмотреть файл

@ -11,6 +11,11 @@
#include <src/include/pmix_config.h>
#include <unistd.h>
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include <pmix_common.h>
#include "src/include/pmix_socket_errno.h"
@ -19,26 +24,26 @@
#include "src/util/error.h"
#include "src/util/output.h"
#include <unistd.h>
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include "src/mca/psec/psec.h"
#include "psec_native.h"
static int native_init(void);
static pmix_status_t native_init(void);
static void native_finalize(void);
static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred);
static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
char **cred, size_t *len);
static pmix_status_t validate_cred(pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char *cred, size_t len);
pmix_psec_module_t pmix_native_module = {
.name = "native",
.init = native_init,
.finalize = native_finalize,
.create_cred = create_cred,
.validate_cred = validate_cred
};
static int native_init(void)
static pmix_status_t native_init(void)
{
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: native init");
@ -51,7 +56,44 @@ static void native_finalize(void)
"psec: native finalize");
}
static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred)
static pmix_status_t create_cred(pmix_listener_protocol_t protocol,
char **cred, size_t *len)
{
uid_t euid;
gid_t egid;
char *tmp, *ptr;
if (PMIX_PROTOCOL_V1 == protocol ||
PMIX_PROTOCOL_V3 == protocol) {
/* these are usock protocols - nothing to do */
*cred = NULL;
*len = 0;
return PMIX_SUCCESS;
}
if (PMIX_PROTOCOL_V2 == protocol) {
/* tcp protocol - need to provide our effective
* uid and gid for validation on remote end */
tmp = (char*)malloc(sizeof(uid_t) + sizeof(gid_t));
if (NULL == tmp) {
return PMIX_ERR_NOMEM;
}
euid = geteuid();
memcpy(tmp, &euid, sizeof(uid_t));
ptr = tmp + sizeof(uid_t);
egid = getegid();
memcpy(ptr, &egid, sizeof(gid_t));
*cred = tmp;
*len = sizeof(uid_t) + sizeof(gid_t);
return PMIX_SUCCESS;
}
/* unrecognized protocol */
return PMIX_ERR_NOT_SUPPORTED;
}
static pmix_status_t validate_cred(pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char *cred, size_t len)
{
#if defined(SO_PEERCRED)
#ifdef HAVE_STRUCT_SOCKPEERCRED_UID
@ -64,62 +106,104 @@ static pmix_status_t validate_cred(pmix_peer_t *peer, char *cred)
#endif
uid_t euid;
gid_t gid;
char *ptr;
size_t ln;
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: native validate_cred %s", cred ? cred : "NULL");
if (PMIX_PROTOCOL_V1 == protocol ||
PMIX_PROTOCOL_V3 == protocol) {
/* these are usock protocols - get the remote side's uid/gid */
#if defined(SO_PEERCRED) && (defined(HAVE_STRUCT_UCRED_UID) || defined(HAVE_STRUCT_UCRED_CR_UID))
/* Ignore received 'cred' and validate ucred for socket instead. */
pmix_output_verbose(2, pmix_globals.debug_output,
"psec:native checking getsockopt for peer credentials");
if (getsockopt (peer->sd, SOL_SOCKET, SO_PEERCRED, &ucred, &crlen) < 0) {
/* Ignore received 'cred' and validate ucred for socket instead. */
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: getsockopt SO_PEERCRED failed: %s",
strerror (pmix_socket_errno));
pmix_output(0, "ONE");
return PMIX_ERR_INVALID_CRED;
}
"psec:native checking getsockopt on socket %d for peer credentials", peer->sd);
if (getsockopt (peer->sd, SOL_SOCKET, SO_PEERCRED, &ucred, &crlen) < 0) {
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: getsockopt SO_PEERCRED failed: %s",
strerror (pmix_socket_errno));
return PMIX_ERR_INVALID_CRED;
}
#if defined(HAVE_STRUCT_UCRED_UID)
euid = ucred.uid;
gid = ucred.gid;
euid = ucred.uid;
gid = ucred.gid;
#else
euid = ucred.cr_uid;
gid = ucred.cr_gid;
euid = ucred.cr_uid;
gid = ucred.cr_gid;
#endif
#elif defined(HAVE_GETPEEREID)
pmix_output_verbose(2, pmix_globals.debug_output,
"psec:native checking getpeereid for peer credentials");
if (0 != getpeereid(peer->sd, &euid, &gid)) {
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: getsockopt getpeereid failed: %s",
strerror (pmix_socket_errno));
pmix_output(0, "TWO");
return PMIX_ERR_INVALID_CRED;
"psec:native checking getpeereid on socket %d for peer credentials", peer->sd);
if (0 != getpeereid(peer->sd, &euid, &gid)) {
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: getsockopt getpeereid failed: %s",
strerror (pmix_socket_errno));
return PMIX_ERR_INVALID_CRED;
}
#else
pmix_output(0, "FIVE");
return PMIX_ERR_NOT_SUPPORTED;
return PMIX_ERR_NOT_SUPPORTED;
#endif
/* check uid */
if (euid != peer->info->uid) {
/* check uid */
if (euid != peer->info->uid) {
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: socket cred contains invalid uid %u", euid);
return PMIX_ERR_INVALID_CRED;
}
/* check gid */
if (gid != peer->info->gid) {
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: socket cred contains invalid gid %u", gid);
return PMIX_ERR_INVALID_CRED;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: socket cred contains invalid uid %u", euid);
pmix_output(0, "THREE");
return PMIX_ERR_INVALID_CRED;
"psec: native credential %u:%u valid",
euid, gid);
return PMIX_SUCCESS;
}
/* check gid */
if (gid != peer->info->gid) {
if (PMIX_PROTOCOL_V2 == protocol) {
/* this is a tcp protocol, so the cred is actually the uid/gid
* passed upwards from the client */
ln = len;
euid = 0;
gid = 0;
if (sizeof(uid_t) <= ln) {
memcpy(&euid, cred, sizeof(uid_t));
ln -= sizeof(uid_t);
ptr = cred + sizeof(uid_t);
} else {
return PMIX_ERR_INVALID_CRED;
}
if (sizeof(gid_t) <= ln) {
memcpy(&gid, ptr, sizeof(gid_t));
} else {
return PMIX_ERR_INVALID_CRED;
}
/* check uid */
if (euid != peer->info->uid) {
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: socket cred contains invalid uid %u", euid);
return PMIX_ERR_INVALID_CRED;
}
/* check gid */
if (gid != peer->info->gid) {
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: socket cred contains invalid gid %u", gid);
return PMIX_ERR_INVALID_CRED;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: socket cred contains invalid gid %u", gid);
pmix_output(0, "FOUR");
return PMIX_ERR_INVALID_CRED;
"psec: native credential %u:%u valid",
euid, gid);
return PMIX_SUCCESS;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: native credential %u:%u valid",
euid, gid);
return PMIX_SUCCESS;
/* don't recognize the protocol */
return PMIX_ERR_NOT_SUPPORTED;
}

50
opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,50 @@
# -*- makefile -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers = psec_none.h
sources = \
psec_none_component.c \
psec_none.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_pmix_psec_none_DSO
lib =
lib_sources =
component = mca_psec_none.la
component_sources = $(headers) $(sources)
else
lib = libmca_psec_none.la
lib_sources = $(headers) $(sources)
component =
component_sources =
endif
mcacomponentdir = $(pmixlibdir)
mcacomponent_LTLIBRARIES = $(component)
mca_psec_none_la_SOURCES = $(component_sources)
mca_psec_none_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(lib)
libmca_psec_none_la_SOURCES = $(lib_sources)
libmca_psec_none_la_LDFLAGS = -module -avoid-version

63
opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.c Обычный файл
Просмотреть файл

@ -0,0 +1,63 @@
/*
* Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <src/include/pmix_config.h>
#include <pmix_common.h>
#include "src/include/pmix_socket_errno.h"
#include "src/include/pmix_globals.h"
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/output.h"
#include <unistd.h>
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include "src/mca/psec/psec.h"
#include "psec_none.h"
static pmix_status_t none_init(void);
static void none_finalize(void);
static pmix_status_t validate_cred(pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char *cred, size_t len);
pmix_psec_module_t pmix_none_module = {
.name = "none",
.init = none_init,
.finalize = none_finalize,
.validate_cred = validate_cred
};
static pmix_status_t none_init(void)
{
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: none init");
return PMIX_SUCCESS;
}
static void none_finalize(void)
{
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: none finalize");
}
static pmix_status_t validate_cred(pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char *cred, size_t len)
{
pmix_output_verbose(2, pmix_globals.debug_output,
"psec: none always reports valid");
return PMIX_SUCCESS;
}

27
opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none.h Обычный файл
Просмотреть файл

@ -0,0 +1,27 @@
/*
* Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
*
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef PMIX_NATIVE_H
#define PMIX_NATIVE_H
#include <src/include/pmix_config.h>
#include "src/mca/psec/psec.h"
BEGIN_C_DECLS
/* the component must be visible data for the linker to find it */
PMIX_EXPORT extern pmix_psec_base_component_t mca_psec_none_component;
extern pmix_psec_module_t pmix_none_module;
END_C_DECLS
#endif

91
opal/mca/pmix/pmix2x/pmix/src/mca/psec/none/psec_none_component.c Обычный файл
Просмотреть файл

@ -0,0 +1,91 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
* These symbols are in a file by themselves to provide nice linker
* semantics. Since linkers generally pull in symbols by object
* files, keeping these symbols as the only symbols in this file
* prevents utility programs such as "ompi_info" from having to import
* entire components just to query their version and parameters.
*/
#include <src/include/pmix_config.h>
#include "pmix_common.h"
#include "src/mca/psec/psec.h"
#include "psec_none.h"
static pmix_status_t component_open(void);
static pmix_status_t component_close(void);
static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority);
static pmix_psec_module_t* assign_module(void);
/*
* Instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
pmix_psec_base_component_t mca_psec_none_component = {
.base = {
PMIX_PSEC_BASE_VERSION_1_0_0,
/* Component name and version */
.pmix_mca_component_name = "none",
PMIX_MCA_BASE_MAKE_VERSION(component,
PMIX_MAJOR_VERSION,
PMIX_MINOR_VERSION,
PMIX_RELEASE_VERSION),
/* Component open and close functions */
.pmix_mca_open_component = component_open,
.pmix_mca_close_component = component_close,
.pmix_mca_query_component = component_query,
},
.data = {
/* The component is checkpoint ready */
PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT
},
.assign_module = assign_module
};
static int component_open(void)
{
return PMIX_SUCCESS;
}
static int component_query(pmix_mca_base_module_t **module, int *priority)
{
*priority = 0;
*module = (pmix_mca_base_module_t *)&pmix_none_module;
return PMIX_SUCCESS;
}
static int component_close(void)
{
return PMIX_SUCCESS;
}
static pmix_psec_module_t* assign_module(void)
{
return &pmix_none_module;
}

70
opal/mca/pmix/pmix2x/pmix/src/mca/psec/psec.h
Просмотреть файл

@ -33,6 +33,7 @@
#include "src/mca/mca.h"
#include "src/mca/base/pmix_mca_base_var.h"
#include "src/mca/base/pmix_mca_base_framework.h"
#include "src/mca/ptl/ptl_types.h"
BEGIN_C_DECLS
@ -45,7 +46,7 @@ struct pmix_peer_t;
* Initialize the module. Returns an error if the module cannot
* run, success if it can and wants to be used.
*/
typedef int (*pmix_psec_base_module_init_fn_t)(void);
typedef pmix_status_t (*pmix_psec_base_module_init_fn_t)(void);
/**
* Finalize the module. Tear down any allocated storage, disconnect
@ -55,10 +56,12 @@ typedef void (*pmix_psec_base_module_fini_fn_t)(void);
/**** CLIENT-SIDE FUNCTIONS ****/
/**
* Create and return a string representation of a credential for this
* client
* Create and return a credential for this client - this
* could be a string or a byte array, which is why we must
* also return the length
*/
typedef char* (*pmix_psec_base_module_create_cred_fn_t)(void);
typedef pmix_status_t (*pmix_psec_base_module_create_cred_fn_t)(pmix_listener_protocol_t protocol,
char **cred, size_t *len);
/**
* Perform the client-side handshake. Note that it is not required
@ -70,9 +73,12 @@ typedef pmix_status_t (*pmix_psec_base_module_client_hndshk_fn_t)(int sd);
/**** SERVER-SIDE FUNCTIONS ****/
/**
* Validate a client's credential
* Validate a client's credential - the credential could be a string
* or an array of bytes, which is why we include the length
*/
typedef pmix_status_t (*pmix_psec_base_module_validate_cred_fn_t)(struct pmix_peer_t *peer, char *cred);
typedef pmix_status_t (*pmix_psec_base_module_validate_cred_fn_t)(struct pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char *cred, size_t len);
/**
* Perform the server-side handshake. Note that it is not required
@ -98,6 +104,50 @@ typedef struct {
} pmix_psec_module_t;
/* define an API module */
/* get a list of available options - caller must free results
* when done */
typedef char* (*pmix_psec_API_get_available_modules_fn_t)(void);
/* Select a psec module for a given peer */
typedef pmix_status_t (*pmix_psec_API_assign_module_fn_t)(struct pmix_peer_t *peer,
const char *options);
/**
* Create and return a string representation of a credential for this
* client
*/
typedef pmix_status_t (*pmix_psec_API_create_cred_fn_t)(struct pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char **cred, size_t *len);
/**
* Perform the client-side handshake. Note that it is not required
* (and indeed, would be rare) for a protocol to use both the
* credential and handshake interfaces. It is acceptable, therefore,
* for one of them to be NULL */
typedef pmix_status_t (*pmix_psec_API_client_hndshk_fn_t)(struct pmix_peer_t *peer, int sd);
/**** SERVER-SIDE FUNCTIONS ****/
/**
* Validate a client's connection request
*/
typedef pmix_status_t (*pmix_psec_API_validate_connection_fn_t)(struct pmix_peer_t *peer,
pmix_listener_protocol_t protocol,
char *cred, size_t len);
typedef struct {
pmix_psec_API_get_available_modules_fn_t get_available_modules;
pmix_psec_API_assign_module_fn_t assign_module;
pmix_psec_API_create_cred_fn_t create_cred;
pmix_psec_API_client_hndshk_fn_t client_handshake;
pmix_psec_API_validate_connection_fn_t validate_connection;
} pmix_psec_API_t;
PMIX_EXPORT extern pmix_psec_API_t pmix_psec;
/**** COMPONENT STRUCTURE DEFINITION ****/
/* define a component-level API for initializing the component */
@ -122,14 +172,6 @@ struct pmix_psec_base_component_t {
};
typedef struct pmix_psec_base_component_t pmix_psec_base_component_t;
/* Select a psec module for a given peer */
pmix_psec_module_t* pmix_psec_base_assign_module(const char *options);
/* get a list of available options - caller must free results
* when done */
char* pmix_psec_base_get_available_modules(void);
/*
* Macro for use in components that are of type psec
*/

44
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,44 @@
# -*- makefile -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CPPFLAGS = $(LTDLINCL)
# main library setup
noinst_LTLIBRARIES = libmca_ptl.la
libmca_ptl_la_SOURCES =
# local files
headers = ptl.h ptl_types.h
sources =
# Conditionally install the header files
if WANT_INSTALL_HEADERS
pmixdir = $(pmixincludedir)/$(subdir)
nobase_pmix_HEADERS = $(headers)
endif
include base/Makefile.include
libmca_ptl_la_SOURCES += $(headers) $(sources)
distclean-local:
rm -f base/static-components.h

35
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/Makefile.include Обычный файл
Просмотреть файл

@ -0,0 +1,35 @@
# -*- makefile -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
# Copyright (c) 2016 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This makefile.include does not stand on its own - it is included from
# src/Makefile.am
headers += \
base/base.h
sources += \
base/ptl_base_frame.c \
base/ptl_base_select.c \
base/ptl_base_sendrecv.c \
base/ptl_base_listener.c \
base/ptl_base_stubs.c \
base/ptl_base_connect.c

123
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/base.h Обычный файл
Просмотреть файл

@ -0,0 +1,123 @@
/* -*- C -*-
*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
* Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
*/
#ifndef PMIX_PTL_BASE_H_
#define PMIX_PTL_BASE_H_
#include <src/include/pmix_config.h>
#ifdef HAVE_SYS_TIME_H
#include <sys/time.h> /* for struct timeval */
#endif
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#if PMIX_HAVE_HWLOC
#include PMIX_HWLOC_HEADER
#endif
#include "src/class/pmix_pointer_array.h"
#include "src/mca/mca.h"
#include "src/mca/base/pmix_mca_base_framework.h"
#include "src/mca/ptl/ptl.h"
BEGIN_C_DECLS
/*
* MCA Framework
*/
extern pmix_mca_base_framework_t pmix_ptl_base_framework;
/**
* PTL select function
*
* Cycle across available components and construct the list
* of active modules
*/
pmix_status_t pmix_ptl_base_select(void);
/**
* Track an active component
*/
struct pmix_ptl_base_active_t {
pmix_list_item_t super;
pmix_status_t pri;
pmix_ptl_base_component_t *component;
pmix_ptl_module_t *module;
};
typedef struct pmix_ptl_base_active_t pmix_ptl_base_active_t;
PMIX_CLASS_DECLARATION(pmix_ptl_base_active_t);
/* framework globals */
struct pmix_ptl_globals_t {
pmix_list_t actives;
bool initialized;
pmix_list_t posted_recvs; // list of pmix_ptl_posted_recv_t
int stop_thread[2];
bool listen_thread_active;
pmix_list_t listeners;
};
typedef struct pmix_ptl_globals_t pmix_ptl_globals_t;
extern pmix_ptl_globals_t pmix_ptl_globals;
/* API stubs */
PMIX_EXPORT pmix_status_t pmix_ptl_stub_set_notification_cbfunc(pmix_ptl_cbfunc_t cbfunc);
PMIX_EXPORT char* pmix_ptl_stub_get_available_modules(void);
PMIX_EXPORT pmix_status_t pmix_ptl_stub_send_recv(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_cbfunc_t cbfunc,
void *cbdata);
PMIX_EXPORT pmix_status_t pmix_ptl_stub_send_oneway(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_tag_t tag);
PMIX_EXPORT pmix_status_t pmix_ptl_stub_connect_to_peer(struct pmix_peer_t *peer,
pmix_info_t info[], size_t ninfo);
PMIX_EXPORT pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo);
PMIX_EXPORT void pmix_ptl_base_stop_listening(void);
/* base support functions */
PMIX_EXPORT void pmix_ptl_base_send(int sd, short args, void *cbdata);
PMIX_EXPORT void pmix_ptl_base_send_recv(int sd, short args, void *cbdata);
PMIX_EXPORT void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata);
PMIX_EXPORT void pmix_ptl_base_recv_handler(int sd, short flags, void *cbdata);
PMIX_EXPORT void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata);
PMIX_EXPORT pmix_status_t pmix_ptl_base_set_nonblocking(int sd);
PMIX_EXPORT pmix_status_t pmix_ptl_base_set_blocking(int sd);
PMIX_EXPORT pmix_status_t pmix_ptl_base_send_blocking(int sd, char *ptr, size_t size);
PMIX_EXPORT pmix_status_t pmix_ptl_base_recv_blocking(int sd, char *data, size_t size);
PMIX_EXPORT pmix_status_t pmix_ptl_base_connect(struct sockaddr_storage *addr,
pmix_socklen_t len, int *fd);
PMIX_EXPORT void pmix_ptl_base_connection_handler(int sd, short args, void *cbdata);
PMIX_EXPORT pmix_status_t pmix_ptl_base_send_connect_ack(int sd);
PMIX_EXPORT pmix_status_t pmix_ptl_base_recv_connect_ack(int sd);
END_C_DECLS
#endif

249
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_connect.c Обычный файл
Просмотреть файл

@ -0,0 +1,249 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <src/include/pmix_config.h>
#include "include/pmix_stdint.h"
#include <stdio.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#include "include/pmix_socket_errno.h"
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/getid.h"
#include "src/util/strnlen.h"
#include "src/include/pmix_globals.h"
#include "src/client/pmix_client_ops.h"
#include "src/server/pmix_server_ops.h"
#include "src/mca/ptl/base/base.h"
pmix_status_t pmix_ptl_base_set_nonblocking(int sd)
{
int flags;
/* setup the socket as non-blocking */
if ((flags = fcntl(sd, F_GETFL, 0)) < 0) {
pmix_output(0, "ptl:base:set_nonblocking: fcntl(F_GETFL) failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
} else {
flags |= O_NONBLOCK;
if(fcntl(sd, F_SETFL, flags) < 0)
pmix_output(0, "ptl:base:set_nonblocking: fcntl(F_SETFL) failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
}
return PMIX_SUCCESS;
}
pmix_status_t pmix_ptl_base_set_blocking(int sd)
{
int flags;
/* setup the socket as non-blocking */
if ((flags = fcntl(sd, F_GETFL, 0)) < 0) {
pmix_output(0, "ptl:base:set_blocking: fcntl(F_GETFL) failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
} else {
flags &= ~(O_NONBLOCK);
if(fcntl(sd, F_SETFL, flags) < 0)
pmix_output(0, "ptl:base:set_blocking: fcntl(F_SETFL) failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
}
return PMIX_SUCCESS;
}
/*
* A blocking send on a non-blocking socket. Used to send the small amount of connection
* information that identifies the peers endpoint.
*/
pmix_status_t pmix_ptl_base_send_blocking(int sd, char *ptr, size_t size)
{
size_t cnt = 0;
int retval;
pmix_output_verbose(8, pmix_globals.debug_output,
"send blocking of %"PRIsize_t" bytes to socket %d",
size, sd );
while (cnt < size) {
retval = send(sd, (char*)ptr+cnt, size-cnt, 0);
if (retval < 0) {
if (EAGAIN == pmix_socket_errno ||
EWOULDBLOCK == pmix_socket_errno) {
/* just cycle and let it try again */
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking_send received error %d:%s from remote - cycling",
pmix_socket_errno, strerror(pmix_socket_errno));
continue;
}
if (pmix_socket_errno != EINTR) {
pmix_output_verbose(8, pmix_globals.debug_output,
"ptl:base:peer_send_blocking: send() to socket %d failed: %s (%d)\n",
sd, strerror(pmix_socket_errno),
pmix_socket_errno);
return PMIX_ERR_UNREACH;
}
continue;
}
cnt += retval;
}
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking send complete to socket %d", sd);
return PMIX_SUCCESS;
}
/*
* A blocking recv on a non-blocking socket. Used to receive the small amount of connection
* information that identifies the peers endpoint.
*/
pmix_status_t pmix_ptl_base_recv_blocking(int sd, char *data, size_t size)
{
size_t cnt = 0;
pmix_output_verbose(8, pmix_globals.debug_output,
"waiting for blocking recv of %"PRIsize_t" bytes", size);
while (cnt < size) {
int retval = recv(sd, (char *)data+cnt, size-cnt, MSG_WAITALL);
/* remote closed connection */
if (retval == 0) {
pmix_output_verbose(8, pmix_globals.debug_output,
"ptl:base:recv_blocking: remote closed connection");
return PMIX_ERR_UNREACH;
}
/* handle errors */
if (retval < 0) {
if (EAGAIN == pmix_socket_errno ||
EWOULDBLOCK == pmix_socket_errno) {
/* just cycle and let it try again */
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking_recv received error %d:%s from remote - cycling",
pmix_socket_errno, strerror(pmix_socket_errno));
continue;
}
if (pmix_socket_errno != EINTR ) {
/* If we overflow the listen backlog, it's
possible that even though we finished the three
way handshake, the remote host was unable to
transition the connection from half connected
(received the initial SYN) to fully connected
(in the listen backlog). We likely won't see
the failure until we try to receive, due to
timing and the like. The first thing we'll get
in that case is a RST packet, which receive
will turn into a connection reset by peer
errno. In that case, leave the socket in
CONNECT_ACK and propogate the error up to
recv_connect_ack, who will try to establish the
connection again */
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking_recv received error %d:%s from remote - aborting",
pmix_socket_errno, strerror(pmix_socket_errno));
return PMIX_ERR_UNREACH;
}
continue;
}
cnt += retval;
}
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking receive complete from remote");
return PMIX_SUCCESS;
}
#define PMIX_MAX_RETRIES 10
pmix_status_t pmix_ptl_base_connect(struct sockaddr_storage *addr,
pmix_socklen_t addrlen, int *fd)
{
int sd = -1;
int retries = 0;
pmix_output_verbose(2, pmix_globals.debug_output,
"ptl_base_connect: attempting to connect to server");
while (retries < PMIX_MAX_RETRIES) {
retries++;
/* Create the new socket */
sd = socket(addr->ss_family, SOCK_STREAM, 0);
if (sd < 0) {
pmix_output(0, "pmix:create_socket: socket() failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
continue;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"usock_peer_try_connect: attempting to connect to server on socket %d", sd);
/* try to connect */
if (connect(sd, (struct sockaddr*)addr, addrlen) < 0) {
if (pmix_socket_errno == ETIMEDOUT) {
/* The server may be too busy to accept new connections */
pmix_output_verbose(2, pmix_globals.debug_output,
"timeout connecting to server");
CLOSE_THE_SOCKET(sd);
continue;
}
/* Some kernels (Linux 2.6) will automatically software
abort a connection that was ECONNREFUSED on the last
attempt, without even trying to establish the
connection. Handle that case in a semi-rational
way by trying twice before giving up */
if (ECONNABORTED == pmix_socket_errno) {
pmix_output_verbose(2, pmix_globals.debug_output,
"connection to server aborted by OS - retrying");
CLOSE_THE_SOCKET(sd);
continue;
} else {
pmix_output_verbose(2, pmix_globals.debug_output,
"Connect failed: %s (%d)", strerror(pmix_socket_errno),
pmix_socket_errno);
CLOSE_THE_SOCKET(sd);
continue;
}
} else {
/* otherwise, the connect succeeded - so break out of the loop */
break;
}
}
if (retries == PMIX_MAX_RETRIES || sd < 0){
/* We were unsuccessful in establishing this connection, and are
* not likely to suddenly become successful */
if (0 <= sd) {
CLOSE_THE_SOCKET(sd);
}
return PMIX_ERR_UNREACH;
}
*fd = sd;
return PMIX_SUCCESS;
}

220
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_frame.c Обычный файл
Просмотреть файл

@ -0,0 +1,220 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved.
* Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/** @file:
*
*/
#include <src/include/pmix_config.h>
#include <pmix_common.h>
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#include "src/mca/mca.h"
#include "src/mca/base/base.h"
#include "src/mca/base/pmix_mca_base_var.h"
#include "src/mca/base/pmix_mca_base_framework.h"
#include "src/class/pmix_list.h"
#include "src/mca/ptl/base/base.h"
/*
* The following file was created by configure. It contains extern
* statements and the definition of an array of pointers to each
* component's public mca_base_component_t struct.
*/
#include "src/mca/ptl/base/static-components.h"
/* Instantiate the global vars */
pmix_ptl_globals_t pmix_ptl_globals = {{{0}}};
pmix_ptl_API_t pmix_ptl = {
.set_notification_cbfunc = pmix_ptl_stub_set_notification_cbfunc,
.get_available_modules = pmix_ptl_stub_get_available_modules,
.send_recv = pmix_ptl_stub_send_recv,
.send_oneway = pmix_ptl_stub_send_oneway,
.connect_to_peer = pmix_ptl_stub_connect_to_peer,
.start_listening = pmix_ptl_base_start_listening,
.stop_listening = pmix_ptl_base_stop_listening
};
static int pmix_ptl_register(pmix_mca_base_register_flag_t flags)
{
return PMIX_SUCCESS;
}
static pmix_status_t pmix_ptl_close(void)
{
if (!pmix_ptl_globals.initialized) {
return PMIX_SUCCESS;
}
pmix_ptl_globals.initialized = false;
/* ensure the listen thread has been shut down */
pmix_ptl.stop_listening();
/* the components will cleanup when closed */
PMIX_DESTRUCT(&pmix_ptl_globals.actives);
PMIX_LIST_DESTRUCT(&pmix_ptl_globals.posted_recvs);
PMIX_LIST_DESTRUCT(&pmix_ptl_globals.listeners);
return pmix_mca_base_framework_components_close(&pmix_ptl_base_framework, NULL);
}
static pmix_status_t pmix_ptl_open(pmix_mca_base_open_flag_t flags)
{
/* initialize globals */
pmix_ptl_globals.initialized = true;
PMIX_CONSTRUCT(&pmix_ptl_globals.actives, pmix_list_t);
PMIX_CONSTRUCT(&pmix_ptl_globals.posted_recvs, pmix_list_t);
pmix_ptl_globals.listen_thread_active = false;
PMIX_CONSTRUCT(&pmix_ptl_globals.listeners, pmix_list_t);
/* Open up all available components */
return pmix_mca_base_framework_components_open(&pmix_ptl_base_framework, flags);
}
PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, ptl, "PMIx Transfer Layer",
pmix_ptl_register, pmix_ptl_open, pmix_ptl_close,
mca_ptl_base_static_components, 0);
/*** INSTANTIATE INTERNAL CLASSES ***/
PMIX_CLASS_INSTANCE(pmix_ptl_base_active_t,
pmix_list_item_t,
NULL, NULL);
static void scon(pmix_ptl_send_t *p)
{
memset(&p->hdr, 0, sizeof(pmix_ptl_hdr_t));
p->hdr.tag = UINT32_MAX;
p->hdr.nbytes = 0;
p->data = NULL;
p->hdr_sent = false;
p->sdptr = NULL;
p->sdbytes = 0;
}
static void sdes(pmix_ptl_send_t *p)
{
if (NULL != p->data) {
PMIX_RELEASE(p->data);
}
}
PMIX_CLASS_INSTANCE(pmix_ptl_send_t,
pmix_list_item_t,
scon, sdes);
static void rcon(pmix_ptl_recv_t *p)
{
memset(&p->hdr, 0, sizeof(pmix_ptl_hdr_t));
p->hdr.tag = UINT32_MAX;
p->hdr.nbytes = 0;
p->data = NULL;
p->hdr_recvd = false;
p->rdptr = NULL;
p->rdbytes = 0;
}
PMIX_CLASS_INSTANCE(pmix_ptl_recv_t,
pmix_list_item_t,
rcon, NULL);
static void prcon(pmix_ptl_posted_recv_t *p)
{
p->tag = UINT32_MAX;
p->cbfunc = NULL;
p->cbdata = NULL;
}
PMIX_CLASS_INSTANCE(pmix_ptl_posted_recv_t,
pmix_list_item_t,
prcon, NULL);
static void srcon(pmix_ptl_sr_t *p)
{
p->bfr = NULL;
p->cbfunc = NULL;
p->cbdata = NULL;
}
PMIX_CLASS_INSTANCE(pmix_ptl_sr_t,
pmix_object_t,
srcon, NULL);
static void pccon(pmix_pending_connection_t *p)
{
memset(p->nspace, 0, PMIX_MAX_NSLEN+1);
p->info = NULL;
p->ninfo = 0;
p->bfrop = NULL;
p->psec = NULL;
p->ptl = NULL;
p->cred = NULL;
}
static void pcdes(pmix_pending_connection_t *p)
{
if (NULL != p->info) {
PMIX_INFO_FREE(p->info, p->ninfo);
}
if (NULL != p->bfrop) {
free(p->bfrop);
}
if (NULL != p->psec) {
free(p->psec);
}
if (NULL != p->cred) {
free(p->cred);
}
}
PMIX_CLASS_INSTANCE(pmix_pending_connection_t,
pmix_object_t,
pccon, pcdes);
static void lcon(pmix_listener_t *p)
{
p->socket = -1;
p->varname = NULL;
p->uri = NULL;
p->owner_given = false;
p->group_given = false;
p->mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH;
}
static void ldes(pmix_listener_t *p)
{
if (0 <= p->socket) {
CLOSE_THE_SOCKET(p->socket);
}
if (NULL != p->varname) {
free(p->varname);
}
if (NULL != p->uri) {
free(p->uri);
}
}
PMIX_CLASS_INSTANCE(pmix_listener_t,
pmix_list_item_t,
lcon, ldes);
PMIX_CLASS_INSTANCE(pmix_ptl_queue_t,
pmix_object_t,
NULL, NULL);

281
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_listener.c Обычный файл
Просмотреть файл

@ -0,0 +1,281 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
* All rights reserved.
* Copyright (c) 2016 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <src/include/pmix_config.h>
#include <src/include/types.h>
#include <src/include/pmix_stdint.h>
#include <src/include/pmix_socket_errno.h>
#include <pmix_server.h>
#include "src/include/pmix_globals.h"
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#include <fcntl.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_UN_H
#include <sys/un.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include <ctype.h>
#include <sys/stat.h>
#include PMIX_EVENT_HEADER
#include <pthread.h>
#include "src/class/pmix_list.h"
#include "src/util/error.h"
#include "src/util/fd.h"
#include "src/util/output.h"
#include "src/util/pmix_environ.h"
#include "src/mca/ptl/base/base.h"
// local functions for connection support
static void* listen_thread(void *obj);
static pthread_t engine;
static bool setup_complete = false;
/* Cycle across all available plugins and provide them with
* an opportunity to register rendezvous points (server-side
* function). Function is to return PMIX_SUCCESS if at least
* one rendezvous can be defined. */
static pmix_status_t setup_listeners(pmix_info_t *info, size_t ninfo, bool *need_listener)
{
pmix_ptl_base_active_t *active;
pmix_status_t rc;
if (!pmix_ptl_globals.initialized) {
return PMIX_ERR_INIT;
}
PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
if (NULL != active->component->setup_listener) {
rc = active->component->setup_listener(info, ninfo, need_listener);
if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) {
return rc;
}
}
}
/* we must have at least one listener */
if (0 == pmix_list_get_size(&pmix_ptl_globals.listeners)) {
return PMIX_ERR_INIT;
}
return PMIX_SUCCESS;
}
/*
* start listening thread
*/
pmix_status_t pmix_ptl_base_start_listening(pmix_info_t *info, size_t ninfo)
{
pmix_status_t rc;
bool need_listener = false;
/* setup the listeners */
if (!setup_complete) {
if (PMIX_SUCCESS != (rc = setup_listeners(info, ninfo, &need_listener))) {
return rc;
}
}
setup_complete = true;
/* if we don't need a listener thread, then we are done */
if (!need_listener) {
return PMIX_SUCCESS;
}
/*** spawn internal listener thread */
if (0 > pipe(pmix_ptl_globals.stop_thread)) {
PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO);
return PMIX_ERR_OUT_OF_RESOURCE;
}
/* Make sure the pipe FDs are set to close-on-exec so that
they don't leak into children */
if (pmix_fd_set_cloexec(pmix_ptl_globals.stop_thread[0]) != PMIX_SUCCESS ||
pmix_fd_set_cloexec(pmix_ptl_globals.stop_thread[1]) != PMIX_SUCCESS) {
PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO);
close(pmix_ptl_globals.stop_thread[0]);
close(pmix_ptl_globals.stop_thread[1]);
return PMIX_ERR_OUT_OF_RESOURCE;
}
/* fork off the listener thread */
pmix_ptl_globals.listen_thread_active = true;
if (0 > pthread_create(&engine, NULL, listen_thread, NULL)) {
pmix_ptl_globals.listen_thread_active = false;
return PMIX_ERROR;
}
return PMIX_SUCCESS;
}
void pmix_ptl_base_stop_listening(void)
{
int i;
pmix_listener_t *lt;
pmix_output_verbose(8, pmix_globals.debug_output,
"listen_thread: shutdown");
if (!pmix_ptl_globals.listen_thread_active) {
/* nothing we can do */
return;
}
/* mark it as inactive */
pmix_ptl_globals.listen_thread_active = false;
/* use the block to break it loose just in
* case the thread is blocked in a call to select for
* a long time */
i=1;
if (0 > write(pmix_ptl_globals.stop_thread[1], &i, sizeof(int))) {
return;
}
/* wait for thread to exit */
pthread_join(engine, NULL);
/* close the sockets to remove the connection points */
PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) {
CLOSE_THE_SOCKET(lt->socket);
lt->socket = -1;
}
}
static void* listen_thread(void *obj)
{
int rc, max, accepted_connections;
socklen_t addrlen = sizeof(struct sockaddr_storage);
pmix_pending_connection_t *pending_connection;
struct timeval timeout;
fd_set readfds;
pmix_listener_t *lt;
pmix_output_verbose(8, pmix_globals.debug_output,
"listen_thread: active");
while (pmix_ptl_globals.listen_thread_active) {
FD_ZERO(&readfds);
max = -1;
PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) {
FD_SET(lt->socket, &readfds);
max = (lt->socket > max) ? lt->socket : max;
}
/* add the stop_thread fd */
FD_SET(pmix_ptl_globals.stop_thread[0], &readfds);
max = (pmix_ptl_globals.stop_thread[0] > max) ? pmix_ptl_globals.stop_thread[0] : max;
/* set timeout interval */
timeout.tv_sec = 2;
timeout.tv_usec = 0;
/* Block in a select to avoid hammering the cpu. If a connection
* comes in, we'll get woken up right away.
*/
rc = select(max + 1, &readfds, NULL, NULL, &timeout);
if (!pmix_ptl_globals.listen_thread_active) {
/* we've been asked to terminate */
close(pmix_ptl_globals.stop_thread[0]);
close(pmix_ptl_globals.stop_thread[1]);
return NULL;
}
if (rc < 0) {
continue;
}
/* Spin accepting connections until all active listen sockets
* do not have any incoming connections, pushing each connection
* onto the event queue for processing
*/
do {
accepted_connections = 0;
PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) {
/* according to the man pages, select replaces the given descriptor
* set with a subset consisting of those descriptors that are ready
* for the specified operation - in this case, a read. So we need to
* first check to see if this file descriptor is included in the
* returned subset
*/
if (0 == FD_ISSET(lt->socket, &readfds)) {
/* this descriptor is not included */
continue;
}
/* this descriptor is ready to be read, which means a connection
* request has been received - so harvest it. All we want to do
* here is accept the connection and push the info onto the event
* library for subsequent processing - we don't want to actually
* process the connection here as it takes too long, and so the
* OS might start rejecting connections due to timeout.
*/
pending_connection = PMIX_NEW(pmix_pending_connection_t);
pending_connection->protocol = lt->protocol;
pending_connection->ptl = lt->ptl;
event_assign(&pending_connection->ev, pmix_globals.evbase, -1,
EV_WRITE, lt->cbfunc, pending_connection);
pending_connection->sd = accept(lt->socket,
(struct sockaddr*)&(pending_connection->addr),
&addrlen);
if (pending_connection->sd < 0) {
PMIX_RELEASE(pending_connection);
if (pmix_socket_errno != EAGAIN ||
pmix_socket_errno != EWOULDBLOCK) {
if (EMFILE == pmix_socket_errno ||
ENOBUFS == pmix_socket_errno ||
ENOMEM == pmix_socket_errno) {
PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
} else if (EINVAL == pmix_socket_errno ||
EINTR == pmix_socket_errno) {
/* race condition at finalize */
goto done;
} else if (ECONNABORTED == pmix_socket_errno) {
/* they aborted the attempt */
continue;
} else {
pmix_output(0, "listen_thread: accept() failed: %s (%d).",
strerror(pmix_socket_errno), pmix_socket_errno);
}
goto done;
}
continue;
}
pmix_output_verbose(8, pmix_globals.debug_output,
"listen_thread: new connection: (%d, %d)",
pending_connection->sd, pmix_socket_errno);
/* activate the event */
event_active(&pending_connection->ev, EV_WRITE, 1);
accepted_connections++;
}
} while (accepted_connections > 0);
}
done:
pmix_ptl_globals.listen_thread_active = false;
return NULL;
}

94
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_select.c Обычный файл
Просмотреть файл

@ -0,0 +1,94 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <src/include/pmix_config.h>
#include <pmix_common.h>
#include <string.h>
#include "src/mca/mca.h"
#include "src/mca/base/base.h"
#include "src/mca/ptl/base/base.h"
static bool selected = false;
/* Function for selecting a prioritized list of components
* from all those that are available. */
int pmix_ptl_base_select(void)
{
pmix_mca_base_component_list_item_t *cli = NULL;
pmix_ptl_base_component_t *component = NULL;
pmix_ptl_base_active_t *newactive, *active;
pmix_mca_base_module_t *mod;
int pri;
bool inserted;
if (selected) {
/* ensure we don't do this twice */
return PMIX_SUCCESS;
}
selected = true;
/* Query all available components and ask if they have a module */
PMIX_LIST_FOREACH(cli, &pmix_ptl_base_framework.framework_components, pmix_mca_base_component_list_item_t) {
component = (pmix_ptl_base_component_t *) cli->cli_component;
pmix_output_verbose(5, pmix_ptl_base_framework.framework_output,
"mca:ptl:select: checking available component %s",
component->base.pmix_mca_component_name);
/* get the module for this component */
if (PMIX_SUCCESS != component->base.pmix_mca_query_component(&mod, &pri)) {
continue;
}
/* add to our prioritized list of available actives */
newactive = PMIX_NEW(pmix_ptl_base_active_t);
newactive->pri = component->priority;
newactive->component = component;
newactive->module = (pmix_ptl_module_t*)mod;
/* maintain priority order */
inserted = false;
PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
if (newactive->pri > active->pri) {
pmix_list_insert_pos(&pmix_ptl_globals.actives,
(pmix_list_item_t*)active, &newactive->super);
inserted = true;
break;
}
}
if (!inserted) {
/* must be lowest priority - add to end */
pmix_list_append(&pmix_ptl_globals.actives, &newactive->super);
}
}
if (4 < pmix_output_get_verbosity(pmix_ptl_base_framework.framework_output)) {
pmix_output(0, "Final PTL priorities");
/* show the prioritized list */
PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
pmix_output(0, "\tPTL: %s Priority: %d",
active->component->base.pmix_mca_component_name, active->pri);
}
}
return PMIX_SUCCESS;;
}

142
opal/mca/pmix/pmix2x/pmix/src/usock/usock_sendrecv.c → opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c
Просмотреть файл

@ -44,7 +44,7 @@
#include "src/server/pmix_server_ops.h"
#include "src/util/error.h"
#include "usock.h"
#include "src/mca/ptl/base/base.h"
static uint32_t current_tag = 1; // 0 is reserved for system purposes
@ -168,7 +168,7 @@ static pmix_status_t send_bytes(int sd, char **buf, size_t *remain)
goto exit;
}
/* we hit an error and cannot progress this message */
pmix_output(0, "pmix_usock_msg_send_bytes: write failed: %s (%d) [sd = %d]",
pmix_output(0, "pmix_ptl_base_msg_send_bytes: write failed: %s (%d) [sd = %d]",
strerror(pmix_socket_errno),
pmix_socket_errno, sd);
ret = PMIX_ERR_UNREACH;
@ -216,7 +216,7 @@ static pmix_status_t read_bytes(int sd, char **buf, size_t *remain)
* to abort this message
*/
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix_usock_msg_recv: readv failed: %s (%d)",
"pmix_ptl_base_msg_recv: readv failed: %s (%d)",
strerror(pmix_socket_errno),
pmix_socket_errno);
ret = PMIX_ERR_UNREACH;
@ -240,25 +240,26 @@ exit:
* A file descriptor is available/ready for send. Check the state
* of the socket and take the appropriate action.
*/
void pmix_usock_send_handler(int sd, short flags, void *cbdata)
void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata)
{
pmix_peer_t *peer = (pmix_peer_t*)cbdata;
pmix_usock_send_t *msg = peer->send_msg;
pmix_ptl_send_t *msg = peer->send_msg;
pmix_status_t rc;
pmix_output_verbose(2, pmix_globals.debug_output,
"sock:send_handler SENDING TO PEER %s:%d tag %d with %s msg",
"ptl:base:send_handler SENDING TO PEER %s:%d tag %u with %s msg",
peer->info->nptr->nspace, peer->info->rank,
(NULL == msg) ? UINT_MAX : msg->hdr.tag,
(NULL == msg) ? UINT_MAX : ntohl(msg->hdr.tag),
(NULL == msg) ? "NULL" : "NON-NULL");
if (NULL != msg) {
if (!msg->hdr_sent) {
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:send_handler SENDING HEADER");
"ptl:base:send_handler SENDING HEADER");
if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) {
/* header is completely sent */
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:send_handler HEADER SENT");
"ptl:base:send_handler HEADER SENT");
msg->hdr_sent = true;
/* setup to send the data */
if (NULL == msg->data) {
@ -269,14 +270,14 @@ void pmix_usock_send_handler(int sd, short flags, void *cbdata)
} else {
/* send the data as a single block */
msg->sdptr = msg->data->base_ptr;
msg->sdbytes = msg->hdr.nbytes;
msg->sdbytes = ntohl(msg->hdr.nbytes);
}
/* fall thru and let the send progress */
} else if (PMIX_ERR_RESOURCE_BUSY == rc ||
PMIX_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:send_handler RES BUSY OR WOULD BLOCK");
"ptl:base:send_handler RES BUSY OR WOULD BLOCK");
return;
} else {
// report the error
@ -291,22 +292,22 @@ void pmix_usock_send_handler(int sd, short flags, void *cbdata)
if (msg->hdr_sent) {
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:send_handler SENDING BODY OF MSG");
"ptl:base:send_handler SENDING BODY OF MSG");
if (PMIX_SUCCESS == (rc = send_bytes(peer->sd, &msg->sdptr, &msg->sdbytes))) {
// message is complete
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:send_handler BODY SENT");
"ptl:base:send_handler BODY SENT");
PMIX_RELEASE(msg);
peer->send_msg = NULL;
} else if (PMIX_ERR_RESOURCE_BUSY == rc ||
PMIX_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:send_handler RES BUSY OR WOULD BLOCK");
"ptl:base:send_handler RES BUSY OR WOULD BLOCK");
return;
} else {
// report the error
pmix_output(0, "pmix_usock_peer_send_handler: unable to send message ON SOCKET %d",
pmix_output(0, "ptl:base:peer_send_handler: unable to send message ON SOCKET %d",
peer->sd);
event_del(&peer->send_event);
peer->send_ev_active = false;
@ -324,7 +325,7 @@ void pmix_usock_send_handler(int sd, short flags, void *cbdata)
* wait for another send_event to fire before doing so. This gives
* us a chance to service any pending recvs.
*/
peer->send_msg = (pmix_usock_send_t*)
peer->send_msg = (pmix_ptl_send_t*)
pmix_list_remove_first(&peer->send_queue);
}
@ -340,14 +341,17 @@ void pmix_usock_send_handler(int sd, short flags, void *cbdata)
* of the connection with the peer.
*/
void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
void pmix_ptl_base_recv_handler(int sd, short flags, void *cbdata)
{
pmix_status_t rc;
pmix_peer_t *peer = (pmix_peer_t*)cbdata;
pmix_usock_recv_t *msg = NULL;
pmix_ptl_recv_t *msg = NULL;
pmix_ptl_hdr_t hdr;
size_t nbytes;
char *ptr;
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:recv:handler called with peer %s:%d",
"ptl:base:recv:handler called with peer %s:%d",
(NULL == peer) ? "NULL" : peer->info->nptr->nspace,
(NULL == peer) ? PMIX_RANK_UNDEF : peer->info->rank);
@ -357,26 +361,36 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
/* allocate a new message and setup for recv */
if (NULL == peer->recv_msg) {
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:recv:handler allocate new recv msg");
peer->recv_msg = PMIX_NEW(pmix_usock_recv_t);
"ptl:base:recv:handler allocate new recv msg");
peer->recv_msg = PMIX_NEW(pmix_ptl_recv_t);
if (NULL == peer->recv_msg) {
pmix_output(0, "usock_recv_handler: unable to allocate recv message\n");
pmix_output(0, "sptl:base:recv_handler: unable to allocate recv message\n");
goto err_close;
}
peer->recv_msg->peer = peer; // provide a handle back to the peer object
/* start by reading the header */
peer->recv_msg->rdptr = (char*)&peer->recv_msg->hdr;
peer->recv_msg->rdbytes = sizeof(pmix_usock_hdr_t);
peer->recv_msg->rdbytes = sizeof(pmix_ptl_hdr_t);
}
msg = peer->recv_msg;
msg->sd = sd;
/* if the header hasn't been completely read, read it */
if (!msg->hdr_recvd) {
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:recv:handler read hdr on socket %d", peer->sd);
if (PMIX_SUCCESS == (rc = read_bytes(peer->sd, &msg->rdptr, &msg->rdbytes))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"ptl:base:recv:handler read hdr on socket %d", peer->sd);
nbytes = sizeof(pmix_ptl_hdr_t);
ptr = (char*)&hdr;
if (PMIX_SUCCESS == (rc = read_bytes(peer->sd, &ptr, &nbytes))) {
/* completed reading the header */
peer->recv_msg->hdr_recvd = true;
/* convert the hdr to host format */
peer->recv_msg->hdr.pindex = ntohl(hdr.pindex);
peer->recv_msg->hdr.tag = ntohl(hdr.tag);
peer->recv_msg->hdr.nbytes = ntohl(hdr.nbytes);
pmix_output_verbose(2, pmix_globals.debug_output,
"RECVD MSG FOR TAG %d SIZE %d",
(int)peer->recv_msg->hdr.tag,
(int)peer->recv_msg->hdr.nbytes);
/* if this is a zero-byte message, then we are done */
if (0 == peer->recv_msg->hdr.nbytes) {
pmix_output_verbose(2, pmix_globals.debug_output,
@ -388,7 +402,7 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
peer->recv_msg->rdbytes = 0;
} else {
pmix_output_verbose(2, pmix_globals.debug_output,
"usock:recv:handler allocate data region of size %lu",
"ptl:base:recv:handler allocate data region of size %lu",
(unsigned long)peer->recv_msg->hdr.nbytes);
/* allocate the data region */
peer->recv_msg->data = (char*)malloc(peer->recv_msg->hdr.nbytes);
@ -407,7 +421,7 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
* and let the caller know
*/
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix_usock_msg_recv: peer closed connection");
"ptl:base:msg_recv: peer closed connection");
goto err_close;
}
}
@ -436,7 +450,7 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
* and let the caller know
*/
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix_usock_msg_recv: peer closed connection");
"ptl:base:msg_recv: peer closed connection");
goto err_close;
}
}
@ -459,11 +473,44 @@ void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
lost_connection(peer, PMIX_ERR_UNREACH);
}
void pmix_usock_send_recv(int fd, short args, void *cbdata)
void pmix_ptl_base_send(int sd, short args, void *cbdata)
{
pmix_usock_sr_t *ms = (pmix_usock_sr_t*)cbdata;
pmix_usock_posted_recv_t *req;
pmix_usock_send_t *snd;
pmix_ptl_queue_t *queue = (pmix_ptl_queue_t*)cbdata;
pmix_ptl_send_t *snd;
pmix_output_verbose(2, pmix_globals.debug_output,
"[%s:%d] queue callback called: reply to %s:%d on tag %d",
__FILE__, __LINE__,
(queue->peer)->info->nptr->nspace,
(queue->peer)->info->rank, (queue->tag));
snd = PMIX_NEW(pmix_ptl_send_t);
snd->hdr.pindex = htonl(pmix_globals.pindex);
snd->hdr.tag = htonl(queue->tag);
snd->hdr.nbytes = htonl((queue->buf)->bytes_used);
snd->data = (queue->buf);
/* always start with the header */
snd->sdptr = (char*)&snd->hdr;
snd->sdbytes = sizeof(pmix_ptl_hdr_t);
/* if there is no message on-deck, put this one there */
if (NULL == (queue->peer)->send_msg) {
(queue->peer)->send_msg = snd;
} else {
/* add it to the queue */
pmix_list_append(&(queue->peer)->send_queue, &snd->super);
}
/* ensure the send event is active */
if (!(queue->peer)->send_ev_active) {
event_add(&(queue->peer)->send_event, 0);
(queue->peer)->send_ev_active = true;
}
PMIX_RELEASE(queue);
}
void pmix_ptl_base_send_recv(int fd, short args, void *cbdata)
{
pmix_ptl_sr_t *ms = (pmix_ptl_sr_t*)cbdata;
pmix_ptl_posted_recv_t *req;
pmix_ptl_send_t *snd;
uint32_t tag;
/* set the tag */
@ -471,7 +518,7 @@ void pmix_usock_send_recv(int fd, short args, void *cbdata)
if (NULL != ms->cbfunc) {
/* if a callback msg is expected, setup a recv for it */
req = PMIX_NEW(pmix_usock_posted_recv_t);
req = PMIX_NEW(pmix_ptl_posted_recv_t);
/* take the next tag in the sequence */
if (UINT32_MAX == current_tag ) {
current_tag = 1;
@ -484,17 +531,20 @@ void pmix_usock_send_recv(int fd, short args, void *cbdata)
/* add it to the list of recvs - we cannot have unexpected messages
* in this subsystem as the server never sends us something that
* we didn't previously request */
pmix_list_prepend(&pmix_usock_globals.posted_recvs, &req->super);
pmix_list_prepend(&pmix_ptl_globals.posted_recvs, &req->super);
}
snd = PMIX_NEW(pmix_usock_send_t);
snd->hdr.pindex = pmix_globals.pindex;
snd->hdr.tag = tag;
snd->hdr.nbytes = ms->bfr->bytes_used;
pmix_output_verbose(2, pmix_globals.debug_output,
"QUEIENG MSG TO SERVER OF SIZE %d",
(int)ms->bfr->bytes_used);
snd = PMIX_NEW(pmix_ptl_send_t);
snd->hdr.pindex = htonl(pmix_globals.pindex);
snd->hdr.tag = htonl(tag);
snd->hdr.nbytes = htonl(ms->bfr->bytes_used);
snd->data = ms->bfr;
/* always start with the header */
snd->sdptr = (char*)&snd->hdr;
snd->sdbytes = sizeof(pmix_usock_hdr_t);
snd->sdbytes = sizeof(pmix_ptl_hdr_t);
/* if there is no message on-deck, put this one there */
if (NULL == ms->peer->send_msg) {
@ -512,10 +562,10 @@ void pmix_usock_send_recv(int fd, short args, void *cbdata)
PMIX_RELEASE(ms);
}
void pmix_usock_process_msg(int fd, short flags, void *cbdata)
void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata)
{
pmix_usock_recv_t *msg = (pmix_usock_recv_t*)cbdata;
pmix_usock_posted_recv_t *rcv;
pmix_ptl_recv_t *msg = (pmix_ptl_recv_t*)cbdata;
pmix_ptl_posted_recv_t *rcv;
pmix_buffer_t buf;
pmix_output_verbose(5, pmix_globals.debug_output,
@ -523,7 +573,7 @@ void pmix_usock_process_msg(int fd, short flags, void *cbdata)
(int)msg->hdr.nbytes, msg->hdr.tag, msg->sd);
/* see if we have a waiting recv for this message */
PMIX_LIST_FOREACH(rcv, &pmix_usock_globals.posted_recvs, pmix_usock_posted_recv_t) {
PMIX_LIST_FOREACH(rcv, &pmix_ptl_globals.posted_recvs, pmix_ptl_posted_recv_t) {
pmix_output_verbose(5, pmix_globals.debug_output,
"checking msg on tag %u for tag %u",
msg->hdr.tag, rcv->tag);
@ -545,7 +595,7 @@ void pmix_usock_process_msg(int fd, short flags, void *cbdata)
PMIX_DESTRUCT(&buf); // free's the msg data
/* also done with the recv, if not a wildcard or the error tag */
if (UINT32_MAX != rcv->tag && 0 != rcv->tag) {
pmix_list_remove_item(&pmix_usock_globals.posted_recvs, &rcv->super);
pmix_list_remove_item(&pmix_ptl_globals.posted_recvs, &rcv->super);
PMIX_RELEASE(rcv);
}
PMIX_RELEASE(msg);
@ -555,7 +605,7 @@ void pmix_usock_process_msg(int fd, short flags, void *cbdata)
}
/* we get here if no matching recv was found - this is an error */
pmix_output(0, "UNEXPECTED MESSAGE tag =%d", msg->hdr.tag);
pmix_output(0, "UNEXPECTED MESSAGE tag = %d", msg->hdr.tag);
PMIX_RELEASE(msg);
PMIX_REPORT_EVENT(PMIX_ERROR, _notify_complete);
}

107
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/base/ptl_base_stubs.c Обычный файл
Просмотреть файл

@ -0,0 +1,107 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <src/include/pmix_config.h>
#include <stdio.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/include/pmix_globals.h"
#include "src/mca/ptl/base/base.h"
pmix_status_t pmix_ptl_stub_set_notification_cbfunc(pmix_ptl_cbfunc_t cbfunc)
{
pmix_ptl_posted_recv_t *req;
/* post a persistent recv for the special 0 tag so the client can recv
* error notifications from the server */
req = PMIX_NEW(pmix_ptl_posted_recv_t);
if (NULL == req) {
return PMIX_ERR_NOMEM;
}
req->tag = 0;
req->cbfunc = cbfunc;
pmix_output_verbose(5, pmix_globals.debug_output,
"posting notification recv on tag %d", req->tag);
/* add it to the list of recvs - we cannot have unexpected messages
* in this subsystem as the server never sends us something that
* we didn't previously request */
pmix_list_prepend(&pmix_ptl_globals.posted_recvs, &req->super);
return PMIX_SUCCESS;
}
char* pmix_ptl_stub_get_available_modules(void)
{
pmix_ptl_base_active_t *active;
char **tmp=NULL, *reply=NULL;
if (!pmix_ptl_globals.initialized) {
return NULL;
}
PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
pmix_argv_append_nosize(&tmp, active->component->base.pmix_mca_component_name);
}
if (NULL != tmp) {
reply = pmix_argv_join(tmp, ',');
pmix_argv_free(tmp);
}
return reply;
}
pmix_status_t pmix_ptl_stub_send_recv(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_cbfunc_t cbfunc,
void *cbdata)
{
pmix_peer_t *pr = (pmix_peer_t*)peer;
return pr->compat.ptl->send_recv(peer, bfr, cbfunc, cbdata);
}
pmix_status_t pmix_ptl_stub_send_oneway(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_tag_t tag)
{
pmix_peer_t *pr = (pmix_peer_t*)peer;
return pr->compat.ptl->send(peer, bfr, tag);
}
pmix_status_t pmix_ptl_stub_connect_to_peer(struct pmix_peer_t *peer,
pmix_info_t info[], size_t ninfo)
{
pmix_peer_t *pr = (pmix_peer_t*)peer;
pmix_ptl_base_active_t *active;
PMIX_LIST_FOREACH(active, &pmix_ptl_globals.actives, pmix_ptl_base_active_t) {
if (NULL != active->module->connect_to_peer) {
if (PMIX_SUCCESS == active->module->connect_to_peer(peer, info, ninfo)) {
pr->compat.ptl = active->module;
return PMIX_SUCCESS;
}
}
}
return PMIX_ERR_UNREACH;
}

199
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl.h Обычный файл
Просмотреть файл

@ -0,0 +1,199 @@
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
* Copyright (c) 2013-2016 Intel, Inc. All rights reserved
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2016 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* Data packing subsystem.
*/
#ifndef PMIX_PTL_H_
#define PMIX_PTL_H_
#include <src/include/pmix_config.h>
#include <src/include/types.h>
#include "src/mca/mca.h"
#include "src/mca/base/pmix_mca_base_var.h"
#include "src/mca/base/pmix_mca_base_framework.h"
#include "src/buffer_ops/types.h"
#include "ptl_types.h"
BEGIN_C_DECLS
/* forward declaration */
struct pmix_peer_t;
/* The overall objective of this framework is to provide transport
* options by which a server can communicate with a client:
*
* (a) across different versions of the library - e.g., when the
* connection handshake changes.
*
* (b) using different transports as necessitated by different
* environments.
*
* This is a mult-select framework - i.e., multiple components
* are selected and "active" at the same time. The intent is
* to have one component for each use-case, with the
* expectation that the community will do its best not to revise
* communications in manners that expand components to support (a).
* Thus, new variations should be rare, and only a few components
* will exist.
*
* The framework itself reflects the fact that any given peer
* will utilize only one messaging method.
* Thus, once a peer is identified, it will pass its version string
* to this framework's "assign_module" function, which will then
* pass it to each component until one returns a module capable of
* processing the given version. This module is then "attached" to
* the pmix_peer_t object so it can be used for all subsequent
* communication to/from that peer.
*
* Accordingly, there are two levels of APIs defined for this
* framework:
*
* (a) component level - these allow for init/finalize of the
* component, and assignment of a module to a given peer
* based on the version that peer is using
*
* (b) module level - implement send/recv/etc. Note that the
* module only needs to provide those functions that differ
* from the base functions - they don't need to duplicate
* all that code!
*/
/**** MODULE INTERFACE DEFINITION ****/
/* initialize an active plugin - note that servers may have
* multiple active plugins, while clients can only have one */
typedef pmix_status_t (*pmix_ptl_init_fn_t)(void);
/* finalize an active plugin */
typedef void (*pmix_ptl_finalize_fn_t)(void);
/* (TWO-WAY) send a message to the peer, and get a response delivered
* to the specified callback function. The buffer will be free'd
* at the completion of the send, and the cbfunc will be called
* when the corresponding reply is received */
typedef pmix_status_t (*pmix_ptl_send_recv_fn_t)(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_cbfunc_t cbfunc,
void *cbdata);
/* (ONE-WAY) send a message to the peer. The buffer will be free'd
* at the completion of the send */
typedef pmix_status_t (*pmix_ptl_send_fn_t)(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_tag_t tag);
/* connect to a peer - this is a blocking function
* to establish a connection to a peer. It assigns
* the corresponding module to the peer's compat
* structure for future use */
typedef pmix_status_t (*pmix_ptl_connect_to_peer_fn_t)(struct pmix_peer_t *peer,
pmix_info_t info[], size_t ninfo);
/**
* Base structure for a PTL module
*/
struct pmix_ptl_module_t {
pmix_ptl_init_fn_t init;
pmix_ptl_finalize_fn_t finalize;
pmix_ptl_send_recv_fn_t send_recv;
pmix_ptl_send_fn_t send;
pmix_ptl_connect_to_peer_fn_t connect_to_peer;
};
typedef struct pmix_ptl_module_t pmix_ptl_module_t;
/**** API MODULE DEFINITION ****/
/* set the notification callback function to the provided one */
typedef pmix_status_t (*pmix_ptl_set_notification_cbfunc_fn_t)(pmix_ptl_cbfunc_t cbfunc);
/* get a list of available support - caller must free results
* when done. The list is returned as a comma-delimited string
* of available components in priority order, suitable for
* passing to the assign_module function */
typedef char* (*pmix_ptl_get_available_modules_fn_t)(void);
/* Start listening for PMIx clients (server-side function) */
typedef pmix_status_t (*pmix_ptl_start_listening_fn_t)(pmix_info_t *info, size_t ninfo);
/* Stop listening for PMIx clients and cleanup all rendezvous
* points (server-side function) */
typedef void (*pmix_ptl_stop_listening_fn_t)(void);
typedef struct {
pmix_ptl_set_notification_cbfunc_fn_t set_notification_cbfunc;
pmix_ptl_get_available_modules_fn_t get_available_modules;
pmix_ptl_send_recv_fn_t send_recv;
pmix_ptl_send_fn_t send_oneway;
pmix_ptl_connect_to_peer_fn_t connect_to_peer;
pmix_ptl_start_listening_fn_t start_listening;
pmix_ptl_stop_listening_fn_t stop_listening;
} pmix_ptl_API_t;
PMIX_EXPORT extern pmix_ptl_API_t pmix_ptl;
/**** COMPONENT STRUCTURE DEFINITION ****/
/* define a component-level API for establishing a
* communication rendezvous point for local procs. Each active component
* would be given an opportunity to register a listener with the
* PTL base, and/or to establish their own method for handling
* connection requests. The component sets the need_listener flag
* to true if a listener thread is required - otherwise, it does _not_
* modify this parameter */
typedef pmix_status_t (*pmix_ptl_base_setup_listener_fn_t)(pmix_info_t info[], size_t ninfo,
bool *need_listener);
/*
* the standard component data structure
*/
struct pmix_ptl_base_component_t {
pmix_mca_base_component_t base;
pmix_mca_base_component_data_t data;
int priority;
char* uri;
pmix_ptl_base_setup_listener_fn_t setup_listener;
};
typedef struct pmix_ptl_base_component_t pmix_ptl_base_component_t;
/**** DEFINE SOME GENERAL ACCESS FUNCTIONS ****/
/*
* Macro for use in components that are of type ptl
*/
#define PMIX_PTL_BASE_VERSION_1_0_0 \
PMIX_MCA_BASE_VERSION_1_0_0("ptl", 1, 0, 0)
END_C_DECLS
#endif /* PMIX_PTL_H */

257
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/ptl_types.h Обычный файл
Просмотреть файл

@ -0,0 +1,257 @@
/* -*- C -*-
*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved.
* Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* Buffer management types.
*/
#ifndef PMIX_PTL_TYPES_H_
#define PMIX_PTL_TYPES_H_
#include <src/include/pmix_config.h>
#include "src/include/types.h"
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_UN_H
#include <sys/un.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef HAVE_NET_UIO_H
#include <net/uio.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include PMIX_EVENT_HEADER
#include "src/class/pmix_list.h"
#include "src/util/output.h"
#include "src/buffer_ops/types.h"
BEGIN_C_DECLS
// forward declaration
struct pmix_peer_t;
struct pmix_ptl_module_t;
/**** MESSAGING STRUCTURES ****/
typedef uint32_t pmix_ptl_tag_t;
/* header for messages */
typedef struct {
int32_t pindex;
pmix_ptl_tag_t tag;
size_t nbytes;
} pmix_ptl_hdr_t;
/* define the messaging cbfunc */
typedef void (*pmix_ptl_cbfunc_t)(struct pmix_peer_t *peer,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
/* define a callback function for notifying that server connection
* has completed */
typedef void (*pmix_ptl_connect_cbfunc_t)(pmix_status_t status, void *cbdata);
/* define a callback function for processing pending connections */
typedef void (*pmix_ptl_pending_cbfunc_t)(int sd, short args, void *cbdata);
/* structure for sending a message */
typedef struct {
pmix_list_item_t super;
pmix_event_t ev;
pmix_ptl_hdr_t hdr;
pmix_buffer_t *data;
bool hdr_sent;
char *sdptr;
size_t sdbytes;
} pmix_ptl_send_t;
PMIX_CLASS_DECLARATION(pmix_ptl_send_t);
/* structure for recving a message */
typedef struct {
pmix_list_item_t super;
pmix_event_t ev;
struct pmix_peer_t *peer;
int sd;
pmix_ptl_hdr_t hdr;
char *data;
bool hdr_recvd;
char *rdptr;
size_t rdbytes;
} pmix_ptl_recv_t;
PMIX_CLASS_DECLARATION(pmix_ptl_recv_t);
/* structure for tracking posted recvs */
typedef struct {
pmix_list_item_t super;
pmix_event_t ev;
uint32_t tag;
pmix_ptl_cbfunc_t cbfunc;
void *cbdata;
} pmix_ptl_posted_recv_t;
PMIX_CLASS_DECLARATION(pmix_ptl_posted_recv_t);
/* struct for posting send/recv request */
typedef struct {
pmix_object_t super;
volatile bool active;
pmix_event_t ev;
struct pmix_peer_t *peer;
pmix_status_t status;
pmix_buffer_t *bfr;
pmix_ptl_cbfunc_t cbfunc;
void *cbdata;
} pmix_ptl_sr_t;
PMIX_CLASS_DECLARATION(pmix_ptl_sr_t);
typedef struct {
pmix_object_t super;
pmix_event_t ev;
struct pmix_peer_t *peer;
pmix_buffer_t *buf;
pmix_ptl_tag_t tag;
} pmix_ptl_queue_t;
PMIX_CLASS_DECLARATION(pmix_ptl_queue_t);
/* define listener protocol types */
typedef uint16_t pmix_listener_protocol_t;
#define PMIX_PROTOCOL_V1 0 // legacy usock
#define PMIX_PROTOCOL_V2 1 // tcp
#define PMIX_PROTOCOL_V3 2 // updated usock
/* connection support */
typedef struct {
pmix_object_t super;
pmix_event_t ev;
pmix_listener_protocol_t protocol;
int sd;
char nspace[PMIX_MAX_NSLEN+1];
pmix_info_t *info;
size_t ninfo;
pmix_status_t status;
struct sockaddr_storage addr;
char *bfrop;
char *psec;
struct pmix_ptl_module_t *ptl;
pmix_bfrop_buffer_type_t buffer_type;
char *cred;
size_t len;
uid_t uid;
gid_t gid;
} pmix_pending_connection_t;
PMIX_CLASS_DECLARATION(pmix_pending_connection_t);
/* listener objects */
typedef struct pmix_listener_t {
pmix_list_item_t super;
pmix_listener_protocol_t protocol;
struct pmix_ptl_module_t *ptl;
int socket;
char *varname;
char *uri;
uint32_t owner;
bool owner_given;
uint32_t group;
bool group_given;
uint32_t mode;
pmix_ptl_pending_cbfunc_t cbfunc;
} pmix_listener_t;
PMIX_CLASS_DECLARATION(pmix_listener_t);
#define PMIX_ACTIVATE_POST_MSG(ms) \
do { \
pmix_output_verbose(5, pmix_globals.debug_output, \
"[%s:%d] post msg", \
__FILE__, __LINE__); \
event_assign(&((ms)->ev), pmix_globals.evbase, -1, \
EV_WRITE, pmix_ptl_base_process_msg, (ms)); \
event_active(&((ms)->ev), EV_WRITE, 1); \
} while (0)
#define PMIX_SND_CADDY(c, h, s) \
do { \
(c) = PMIX_NEW(pmix_server_caddy_t); \
(void)memcpy(&(c)->hdr, &(h), sizeof(pmix_ptl_hdr_t)); \
PMIX_RETAIN((s)); \
(c)->snd = (s); \
} while (0)
/* queue a message to be sent to one of our procs - must
* provide the following params:
* p - pmix_peer_t of target recipient
* t - tag to be sent to
* b - buffer to be sent
*/
#define PMIX_SERVER_QUEUE_REPLY(p, t, b) \
do { \
pmix_ptl_send_t *snd; \
pmix_output_verbose(5, pmix_globals.debug_output, \
"[%s:%d] queue callback called: reply to %s:%d on tag %d size %d", \
__FILE__, __LINE__, \
(p)->info->nptr->nspace, \
(p)->info->rank, (t), (int)(b)->bytes_used); \
snd = PMIX_NEW(pmix_ptl_send_t); \
snd->hdr.pindex = htonl(pmix_globals.pindex); \
snd->hdr.tag = htonl(t); \
snd->hdr.nbytes = htonl((b)->bytes_used); \
snd->data = (b); \
/* always start with the header */ \
snd->sdptr = (char*)&snd->hdr; \
snd->sdbytes = sizeof(pmix_ptl_hdr_t); \
/* if there is no message on-deck, put this one there */ \
if (NULL == (p)->send_msg) { \
(p)->send_msg = snd; \
} else { \
/* add it to the queue */ \
pmix_list_append(&(p)->send_queue, &snd->super); \
} \
/* ensure the send event is active */ \
if (!(p)->send_ev_active) { \
event_add(&(p)->send_event, 0); \
(p)->send_ev_active = true; \
} \
} while (0)
#define CLOSE_THE_SOCKET(socket) \
do { \
if (0 <= socket) { \
shutdown(socket, 2); \
close(socket); \
socket = -1; \
} \
} while (0)
END_C_DECLS
#endif /* PMIX_PTL_TYPES_H */

50
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,50 @@
# -*- makefile -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers = ptl_tcp.h
sources = \
ptl_tcp_component.c \
ptl_tcp.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_pmix_ptl_tcp_DSO
lib =
lib_sources =
component = mca_ptl_tcp.la
component_sources = $(headers) $(sources)
else
lib = libmca_ptl_tcp.la
lib_sources = $(headers) $(sources)
component =
component_sources =
endif
mcacomponentdir = $(pmixlibdir)
mcacomponent_LTLIBRARIES = $(component)
mca_ptl_tcp_la_SOURCES = $(component_sources)
mca_ptl_tcp_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(lib)
libmca_ptl_tcp_la_SOURCES = $(lib_sources)
libmca_ptl_tcp_la_LDFLAGS = -module -avoid-version

732
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c Обычный файл
Просмотреть файл

@ -0,0 +1,732 @@
/*
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved.
* Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2013-2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
*/
#include <src/include/pmix_config.h>
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#include "src/include/pmix_globals.h"
#include "src/include/pmix_socket_errno.h"
#include "src/client/pmix_client_ops.h"
#include "src/server/pmix_server_ops.h"
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/os_path.h"
#include "src/mca/ptl/base/base.h"
#include "ptl_tcp.h"
static pmix_status_t init(void);
static void finalize(void);
static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
pmix_info_t *info, size_t ninfo);
static pmix_status_t send_recv(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_cbfunc_t cbfunc,
void *cbdata);
static pmix_status_t send_oneway(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_tag_t tag);
pmix_ptl_module_t pmix_ptl_tcp_module = {
.init = init,
.finalize = finalize,
.send_recv = send_recv,
.send = send_oneway,
.connect_to_peer = connect_to_peer
};
static pmix_status_t recv_connect_ack(int sd);
static pmix_status_t send_connect_ack(int sd);
static pmix_status_t init(void)
{
return PMIX_SUCCESS;
}
static void finalize(void)
{
}
static char *pmix_getline(FILE *fp)
{
char *ret, *buff;
char input[1024];
ret = fgets(input, 1024, fp);
if (NULL != ret) {
input[strlen(input)-1] = '\0'; /* remove newline */
buff = strdup(input);
return buff;
}
return NULL;
}
static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
pmix_info_t *info, size_t ninfo)
{
char *evar, **uri;
char *filename, *host;
FILE *fp;
char *srvr, *p, *p2;
struct sockaddr_in *in;
struct sockaddr_in6 *in6;
pmix_socklen_t len;
int sd, rc;
pmix_output_verbose(2, pmix_globals.debug_output,
"ptl:tcp: connecting to server");
/* see if the connection info is in the info array - if
* so, then that overrides all other options */
/* if I am a client, then we need to look for the appropriate
* connection info in the environment */
if (PMIX_PROC_CLIENT == pmix_globals.proc_type) {
if (NULL == (evar = getenv("PMIX_SERVER_URI2"))) {
/* not us */
return PMIX_ERR_NOT_SUPPORTED;
}
/* the URI consists of elements:
* - server nspace.rank
* - ptl rendezvous URI
*/
uri = pmix_argv_split(evar, ';');
if (2 != pmix_argv_count(uri)) {
PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
pmix_argv_free(uri);
return PMIX_ERR_NOT_SUPPORTED;
}
/* set the server nspace */
p = uri[0];
p2 = strchr(p, '.');
*p2 = '\0';
++p2;
pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t);
pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t);
(void)strncpy(pmix_client_globals.myserver.info->nptr->nspace, p, PMIX_MAX_NSLEN);
/* set the server rank */
pmix_client_globals.myserver.info->rank = strtoull(p2, NULL, 10);
/* save the URI, but do not overwrite what we may have received from
* the info-key directives */
if (NULL == mca_ptl_tcp_component.super.uri) {
mca_ptl_tcp_component.super.uri = strdup(uri[1]);
}
pmix_argv_free(uri);
} else if (PMIX_PROC_TOOL == pmix_globals.proc_type) {
/* if we already have a URI, then look no further */
if (NULL == mca_ptl_tcp_component.super.uri) {
/* we have to discover the connection info,
* if possible. Start by looking for the connection
* info in the expected place - if the server supports
* tool connections via TCP, then there will be a
* "contact.txt" file under the system tmpdir */
filename = pmix_os_path(false, mca_ptl_tcp_component.tmpdir, "pmix-contact.txt", NULL);
if (NULL == filename) {
return PMIX_ERR_NOMEM;
}
fp = fopen(filename, "r");
if (NULL == fp) {
/* if we cannot open the file, then the server must not
* be configured to support tool connections - so abort */
free(filename);
return PMIX_ERR_UNREACH;
}
free(filename);
/* get the URI */
srvr = pmix_getline(fp);
if (NULL == srvr) {
PMIX_ERROR_LOG(PMIX_ERR_FILE_READ_FAILURE);
fclose(fp);
return PMIX_ERR_UNREACH;
}
fclose(fp);
/* up to the first ';' is the server nspace/rank */
if (NULL == (p = strchr(srvr, ';'))) {
/* malformed */
free(srvr);
return PMIX_ERR_UNREACH;
}
*p = '\0';
++p; // move past the semicolon
/* the nspace is the section up to the '.' */
if (NULL == (p2 = strchr(srvr, '.'))) {
/* malformed */
free(srvr);
return PMIX_ERR_UNREACH;
}
*p2 = '\0';
++p2;
/* set the server nspace */
pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t);
pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t);
(void)strncpy(pmix_client_globals.myserver.info->nptr->nspace, srvr, PMIX_MAX_NSLEN);
pmix_client_globals.myserver.info->rank = strtoull(p2, NULL, 10);
/* now parse the uti itself */
mca_ptl_tcp_component.super.uri = strdup(p);
free(srvr);
}
}
/* mark that we are the active module for this server */
pmix_client_globals.myserver.compat.ptl = &pmix_ptl_tcp_module;
/* setup the path to the daemon rendezvous point */
memset(&mca_ptl_tcp_component.connection, 0, sizeof(struct sockaddr_storage));
if (0 == strncmp(mca_ptl_tcp_component.super.uri, "tcp4", 4)) {
/* separate the IP address from the port */
p = strdup(mca_ptl_tcp_component.super.uri);
p2 = strchr(&p[7], ':');
*p2 = '\0';
++p2;
host = strdup(&p[7]);
/* load the address */
in = (struct sockaddr_in*)&mca_ptl_tcp_component.connection;
in->sin_family = AF_INET;
in->sin_addr.s_addr = inet_addr(host);
if (in->sin_addr.s_addr == INADDR_NONE) {
return PMIX_ERR_BAD_PARAM;
}
in->sin_port = htons(atoi(p2));
len = sizeof(struct sockaddr_in);
} else {
/* separate the IP address from the port */
p = strdup(mca_ptl_tcp_component.super.uri);
p2 = strchr(&p[7], ':');
*p2 = '\0';
if (']' == p[strlen(p)-1]) {
p[strlen(p)-1] = '\0';
}
if ('[' == p[7]) {
host = strdup(&p[8]);
} else {
host = strdup(&p[7]);
}
/* load the address */
in6 = (struct sockaddr_in6*)&mca_ptl_tcp_component.connection;
in6->sin6_family = AF_INET6;
if (0 == inet_pton(AF_INET6, host, (void*)&in6->sin6_addr)) {
pmix_output (0, "ptl_tcp_parse_uri: Could not convert %s\n", host);
return PMIX_ERR_BAD_PARAM;
}
in6->sin6_port = htons(atoi(p2));
len = sizeof(struct sockaddr_in6);
}
/* establish the connection */
if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&mca_ptl_tcp_component.connection, len, &sd))) {
PMIX_ERROR_LOG(rc);
return rc;
}
pmix_client_globals.myserver.sd = sd;
/* send our identity and any authentication credentials to the server */
if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) {
PMIX_ERROR_LOG(rc);
CLOSE_THE_SOCKET(sd);
return rc;
}
/* do whatever handshake is required */
if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) {
PMIX_ERROR_LOG(rc);
CLOSE_THE_SOCKET(sd);
return rc;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"sock_peer_try_connect: Connection across to server succeeded");
/* mark the connection as made */
pmix_globals.connected = true;
pmix_ptl_base_set_nonblocking(sd);
/* setup recv event */
event_assign(&pmix_client_globals.myserver.recv_event,
pmix_globals.evbase,
pmix_client_globals.myserver.sd,
EV_READ | EV_PERSIST,
pmix_ptl_base_recv_handler, &pmix_client_globals.myserver);
event_add(&pmix_client_globals.myserver.recv_event, 0);
pmix_client_globals.myserver.recv_ev_active = true;
/* setup send event */
event_assign(&pmix_client_globals.myserver.send_event,
pmix_globals.evbase,
pmix_client_globals.myserver.sd,
EV_WRITE|EV_PERSIST,
pmix_ptl_base_send_handler, &pmix_client_globals.myserver);
pmix_client_globals.myserver.send_ev_active = false;
return PMIX_SUCCESS;
}
static pmix_status_t send_recv(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_cbfunc_t cbfunc,
void *cbdata)
{
pmix_ptl_sr_t *ms;
pmix_output_verbose(5, pmix_globals.debug_output,
"[%s:%d] post send to server",
__FILE__, __LINE__);
ms = PMIX_NEW(pmix_ptl_sr_t);
ms->peer = peer;
ms->bfr = bfr;
ms->cbfunc = cbfunc;
ms->cbdata = cbdata;
event_assign(&ms->ev, pmix_globals.evbase, -1,
EV_WRITE, pmix_ptl_base_send_recv, ms);
event_active(&ms->ev, EV_WRITE, 1);
return PMIX_SUCCESS;
}
static pmix_status_t send_oneway(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_tag_t tag)
{
pmix_ptl_queue_t *q;
/* we have to transfer this to an event for thread
* safety as we need to post this message on the
* peer's send queue */
q = PMIX_NEW(pmix_ptl_queue_t);
q->peer = peer;
q->buf = bfr;
q->tag = tag;
event_assign(&q->ev, pmix_globals.evbase, -1,
EV_WRITE, pmix_ptl_base_send, q);
event_active(&q->ev, EV_WRITE, 1);
return PMIX_SUCCESS;
}
/**** SUPPORTING FUNCTIONS ****/
static pmix_status_t send_connect_ack(int sd)
{
char *msg;
pmix_ptl_hdr_t hdr;
size_t sdsize=0, csize=0, len;
char *cred = NULL;
char *sec;
pmix_status_t rc;
uint8_t flag;
uid_t euid;
gid_t egid;
uint32_t u32;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:tcp SEND CONNECT ACK");
/* if we are a server, then we shouldn't be here */
if (PMIX_PROC_IS_SERVER) {
return PMIX_ERR_NOT_SUPPORTED;
}
/* setup the header */
memset(&hdr, 0, sizeof(pmix_ptl_hdr_t));
hdr.pindex = -1;
hdr.tag = UINT32_MAX;
/* a security module was assigned to us during rte_init based
* on a list of available security modules provided by our
* local PMIx server, if known. Now use that module to
* get a credential, if the security system provides one. Not
* every psec module will do so, thus we must first check */
if (PMIX_SUCCESS != (rc = pmix_psec.create_cred(&pmix_client_globals.myserver,
PMIX_PROTOCOL_V2, &cred, &len))) {
return rc;
}
/* allow space for a marker indicating client vs tool */
sdsize = 1;
if (PMIX_PROC_IS_CLIENT) {
flag = 0;
/* reserve space for our nspace and rank info */
sdsize += strlen(pmix_globals.myid.nspace) + 1 + sizeof(int);
} else {
flag = 1;
/* add space for our uid/gid for ACL purposes */
sdsize += 2*sizeof(uint32_t);
}
/* add our active sec module info */
sec = pmix_psec.get_available_modules();
/* set the number of bytes to be read beyond the header */
hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 + sizeof(uint32_t) + len; // must NULL terminate the VERSION string!
/* create a space for our message */
sdsize = (sizeof(hdr) + hdr.nbytes);
if (NULL == (msg = (char*)malloc(sdsize))) {
if (NULL != cred) {
free(cred);
}
free(sec);
return PMIX_ERR_OUT_OF_RESOURCE;
}
memset(msg, 0, sdsize);
/* load the message */
csize=0;
memcpy(msg, &hdr, sizeof(pmix_ptl_hdr_t));
csize += sizeof(pmix_ptl_hdr_t);
/* provide our active psec module */
memcpy(msg+csize, sec, strlen(sec));
csize += strlen(sec)+1;
free(sec);
/* load the length of the credential - we put this in uint32_t
* format as that is a fixed size, and convert to network
* byte order for heterogeneity */
u32 = htonl((uint32_t)len);
memcpy(msg+csize, &u32, sizeof(uint32_t));
csize += sizeof(uint32_t);
/* load the credential */
if (0 < u32) {
memcpy(msg+csize, cred, len);
csize += len;
}
/* load our process type - this is a single byte,
* so no worry about heterogeneity here */
memcpy(msg+csize, &flag, 1);
csize += 1;
if (PMIX_PROC_IS_CLIENT) {
/* if we are a client, provide our nspace/rank */
memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace));
csize += strlen(pmix_globals.myid.nspace)+1;
/* again, need to convert */
u32 = htonl((uint32_t)pmix_globals.myid.rank);
memcpy(msg+csize, &u32, sizeof(uint32_t));
csize += sizeof(uint32_t);
} else {
/* if we are a tool, provide our uid/gid for ACL support - note
* that we have to convert so we can handle heterogeneity */
euid = geteuid();
u32 = htonl(euid);
memcpy(msg+csize, &u32, sizeof(uint32_t));
csize += sizeof(uint32_t);
egid = getegid();
u32 = htonl(egid);
memcpy(msg+csize, &u32, sizeof(uint32_t));
csize += sizeof(uint32_t);
}
/* provide our version */
memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
csize += strlen(PMIX_VERSION)+1;
/* send the entire message across */
if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) {
free(msg);
if (NULL != cred) {
free(cred);
}
return PMIX_ERR_UNREACH;
}
free(msg);
if (NULL != cred) {
free(cred);
}
return PMIX_SUCCESS;
}
/* we receive a connection acknowledgement from the server,
* consisting of nothing more than a status report. If success,
* then we initiate authentication method */
static pmix_status_t recv_connect_ack(int sd)
{
pmix_status_t reply;
pmix_status_t rc;
struct timeval tv, save;
pmix_socklen_t sz;
bool sockopt = true;
uint32_t u32;
pmix_nspace_t *nsptr;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT ACK FROM SERVER");
/* get the current timeout value so we can reset to it */
sz = sizeof(save);
if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) {
if (ENOPROTOOPT == errno) {
sockopt = false;
} else {
return PMIX_ERR_UNREACH;
}
} else {
/* set a timeout on the blocking recv so we don't hang */
tv.tv_sec = 2;
tv.tv_usec = 0;
if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO");
return PMIX_ERR_UNREACH;
}
}
/* receive the status reply */
rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t));
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
return rc;
}
reply = ntohl(u32);
/* if the status indicates an error, then we are done */
if (PMIX_SUCCESS != reply) {
PMIX_ERROR_LOG(reply);
return reply;
}
if (PMIX_PROC_IS_CLIENT) {
/* see if they want us to do the handshake */
if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
if (PMIX_SUCCESS != (rc = pmix_psec.client_handshake(&pmix_client_globals.myserver, sd))) {
return rc;
}
} else if (PMIX_SUCCESS != reply) {
return reply;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT CONFIRMATION");
/* receive our index into the server's client array */
rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t));
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
return rc;
}
pmix_globals.pindex = ntohl(u32);
} else {
/* recv our nspace */
rc = pmix_ptl_base_recv_blocking(sd, (char*)&pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
return rc;
}
/* setup required bookkeeping */
nsptr = PMIX_NEW(pmix_nspace_t);
(void)strncpy(nsptr->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
pmix_list_append(&pmix_globals.nspaces, &nsptr->super);
/* our rank is always zero */
pmix_globals.myid.rank = 0;
/* get the server's nspace and rank so we can send to it */
pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t);
pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t);
pmix_ptl_base_recv_blocking(sd, (char*)pmix_client_globals.myserver.info->nptr->nspace, PMIX_MAX_NSLEN+1);
pmix_ptl_base_recv_blocking(sd, (char*)&(pmix_client_globals.myserver.info->rank), sizeof(int));
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT CONFIRMATION FOR TOOL %s:%d FROM SERVER %s:%d",
pmix_globals.myid.nspace, pmix_globals.myid.rank,
pmix_client_globals.myserver.info->nptr->nspace,
pmix_client_globals.myserver.info->rank);
/* get the returned status from the security handshake */
pmix_ptl_base_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t));
if (PMIX_SUCCESS != reply) {
/* see if they want us to do the handshake */
if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
if (NULL == pmix_psec.client_handshake) {
return PMIX_ERR_HANDSHAKE_FAILED;
}
if (PMIX_SUCCESS != (reply = pmix_psec.client_handshake(&pmix_client_globals.myserver, sd))) {
return reply;
}
/* if the handshake succeeded, then fall thru to the next step */
} else {
return reply;
}
}
}
if (sockopt) {
if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
return PMIX_ERR_UNREACH;
}
}
return PMIX_SUCCESS;
}
#if 0
DIR *cur_dirp = NULL;
struct dirent * dir_entry;
bool connect_to_system_server = false;
bool connect_to_system_first = false;
bool connection_defined = false;
/* scan incoming info for directives */
if (NULL != info) {
for (n=0; n < ninfo; n++) {
if (strcmp(info[n].key, PMIX_SERVER_PIDINFO) == 0) {
server_pid = info[n].value.data.pid;
server_pid_given = true;
} else if (strcmp(info[n].key, PMIX_CONNECT_TO_SYSTEM) == 0) {
connect_to_system_server = info[n].value.data.flag;
connection_defined = true;
} else if (strcmp(info[n].key, PMIX_CONNECT_SYSTEM_FIRST) == 0) {
connect_to_system_first = info[n].value.data.flag;
connection_defined = true;
} else if (strcmp(info[n].key, PMIX_SERVER_TMPDIR) == 0 &&
NULL == mytmpdir) {
mytmpdir = strdup(info[n].value.data.string);
} else if (strcmp(info[n].key, PMIX_SYSTEM_TMPDIR) == 0 &&
NULL == systmpdir) {
systmpdir = strdup(info[n].value.data.string);
}
}
}
/* if we are to connect solely to the system-level daemon,
* or to preferentially connect to the system-level daemon,
* or nothing was specified at all, then look to see if a
* rendezvous point in that location exists */
if (connect_to_system_server || connect_to_system_first || !connection_defined) {
/* find the temp dir */
if (NULL != systmpdir) {
tdir = systmpdir;
} else if (NULL == (tdir = getenv("TMPDIR"))) {
if (NULL == (tdir = getenv("TEMP"))) {
if (NULL == (tdir = getenv("TMP"))) {
tdir = "/tmp";
}
}
}
snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/pmix.sys.%s", tdir, hostname);
/* see if the rendezvous file exists */
if (0 != access(address.sun_path, R_OK)) {
/* if it was a requirement, then error out */
if (connect_to_system_server) {
return PMIX_ERR_UNREACH;
}
/* otherwise, this isn't a fatal error - reset the addr */
memset(&address, 0, sizeof(struct sockaddr_un));
address.sun_family = AF_UNIX;
connection_defined = false;
} else {
/* connect to this server */
connection_defined = true;
}
}
if (!connection_defined) {
/* if we get here, then either we are to connect to
* a non-system daemon, or a system-level daemon was
* not found - so now look for the session daemon */
/* find the temp dir */
if (NULL != mytmpdir) {
tdir = mytmpdir;
} else if (NULL == (tdir = getenv("TMPDIR"))) {
if (NULL == (tdir = getenv("TEMP"))) {
if (NULL == (tdir = getenv("TMP"))) {
tdir = "/tmp";
}
}
}
/* if they gave us a specific pid, then look for that
* particular server - otherwise, see if there is only
* one on this node and default to it */
if (server_pid_given) {
snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/pmix.%s.%d", tdir, hostname, server_pid);
/* if the rendezvous file doesn't exist, that's an error */
if (0 != access(address.sun_path, R_OK)) {
return PMIX_ERR_NOT_FOUND;
}
} else {
/* open up the temp directory */
if (NULL == (cur_dirp = opendir(tdir))) {
return PMIX_ERR_NOT_FOUND;
}
/* search the entries for something that starts with pmix.hostname */
if (0 > asprintf(&tmp, "pmix.%s", hostname)) {
closedir(cur_dirp);
return PMIX_ERR_NOMEM;
}
evar = NULL;
while (NULL != (dir_entry = readdir(cur_dirp))) {
if (0 == strncmp(dir_entry->d_name, tmp, strlen(tmp))) {
/* found one - if more than one, then that's an error */
if (NULL != evar) {
closedir(cur_dirp);
free(evar);
free(tmp);
return PMIX_ERR_INIT;
}
evar = strdup(dir_entry->d_name);
}
}
free(tmp);
closedir(cur_dirp);
if (NULL == evar) {
/* none found */
return PMIX_ERR_INIT;
}
/* use the found one as our contact point */
snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/%s", tdir, evar);
free(evar);
}
}
#endif

56
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.h Обычный файл
Просмотреть файл

@ -0,0 +1,56 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef PMIX_PTL_TCP_H
#define PMIX_PTL_TCP_H
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
#ifdef HAVE_ARPA_INET_H
#include <arpa/inet.h>
#endif
#ifdef HAVE_NETDB_H
#include <netdb.h>
#endif
#include "src/mca/ptl/ptl.h"
BEGIN_C_DECLS
typedef struct {
pmix_ptl_base_component_t super;
char *tmpdir;
char *if_include;
char *if_exclude;
int ipv4_port;
int ipv6_port;
bool disable_ipv4_family;
bool disable_ipv6_family;
struct sockaddr_storage connection;
char *filename;
} pmix_ptl_tcp_component_t;
extern pmix_ptl_tcp_component_t mca_ptl_tcp_component;
extern pmix_ptl_module_t pmix_ptl_tcp_module;
END_C_DECLS
#endif /* PMIX_PTL_TCP_H */

1192
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c Обычный файл
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

50
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,50 @@
# -*- makefile -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved.
# Copyright (c) 2013-2016 Intel, Inc. All rights reserved
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers = ptl_usock.h
sources = \
ptl_usock_component.c \
ptl_usock.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_pmix_ptl_usock_DSO
lib =
lib_sources =
component = mca_ptl_usock.la
component_sources = $(headers) $(sources)
else
lib = libmca_ptl_usock.la
lib_sources = $(headers) $(sources)
component =
component_sources =
endif
mcacomponentdir = $(pmixlibdir)
mcacomponent_LTLIBRARIES = $(component)
mca_ptl_usock_la_SOURCES = $(component_sources)
mca_ptl_usock_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(lib)
libmca_ptl_usock_la_SOURCES = $(lib_sources)
libmca_ptl_usock_la_LDFLAGS = -module -avoid-version

358
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.c Обычный файл
Просмотреть файл

@ -0,0 +1,358 @@
/*
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010-2011 Oak Ridge National Labs. All rights reserved.
* Copyright (c) 2011-2014 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2013-2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
*/
#include <src/include/pmix_config.h>
#include "pmix_common.h"
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_UN_H
#include <sys/un.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/client/pmix_client_ops.h"
#include "src/include/pmix_globals.h"
#include "src/include/pmix_socket_errno.h"
#include "src/mca/psec/psec.h"
#include "src/mca/ptl/base/base.h"
#include "ptl_usock.h"
static pmix_status_t init(void);
static void finalize(void);
static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
pmix_info_t *info, size_t ninfo);
static pmix_status_t send_recv(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_cbfunc_t cbfunc,
void *cbdata);
static pmix_status_t send_oneway(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_tag_t tag);
pmix_ptl_module_t pmix_ptl_usock_module = {
.init = init,
.finalize = finalize,
.send_recv = send_recv,
.send = send_oneway,
.connect_to_peer = connect_to_peer
};
static pmix_status_t recv_connect_ack(int sd);
static pmix_status_t send_connect_ack(int sd);
static pmix_status_t init(void)
{
return PMIX_SUCCESS;
}
static void finalize(void)
{
}
static pmix_status_t connect_to_peer(struct pmix_peer_t *peer,
pmix_info_t *info, size_t ninfo)
{
struct sockaddr_un *address;
char *evar, **uri;
pmix_status_t rc;
int sd;
pmix_socklen_t len;
/* if we are not a client, there is nothing we can do */
if (!PMIX_PROC_IS_CLIENT) {
return PMIX_ERR_NOT_SUPPORTED;
}
/* if we don't have a path to the daemon rendezvous point,
* then we need to return an error */
if (NULL == (evar = getenv("PMIX_SERVER_URI"))) {
/* let the caller know that the server isn't available */
return PMIX_ERR_SERVER_NOT_AVAIL;
}
uri = pmix_argv_split(evar, ':');
if (3 != pmix_argv_count(uri)) {
pmix_argv_free(uri);
return PMIX_ERROR;
}
/* set the server nspace */
pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t);
pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t);
(void)strncpy(pmix_client_globals.myserver.info->nptr->nspace, uri[0], PMIX_MAX_NSLEN);
/* set the server rank */
pmix_client_globals.myserver.info->rank = strtoull(uri[1], NULL, 10);
/* setup the path to the daemon rendezvous point */
memset(&mca_ptl_usock_component.connection, 0, sizeof(struct sockaddr_storage));
address = (struct sockaddr_un*)&mca_ptl_usock_component.connection;
address->sun_family = AF_UNIX;
snprintf(address->sun_path, sizeof(address->sun_path)-1, "%s", uri[2]);
/* if the rendezvous file doesn't exist, that's an error */
if (0 != access(uri[2], R_OK)) {
pmix_argv_free(uri);
return PMIX_ERR_NOT_FOUND;
}
pmix_argv_free(uri);
/* establish the connection */
len = sizeof(struct sockaddr_un);
if (PMIX_SUCCESS != (rc = pmix_ptl_base_connect(&mca_ptl_usock_component.connection, len, &sd))) {
PMIX_ERROR_LOG(rc);
return rc;
}
pmix_client_globals.myserver.sd = sd;
/* send our identity and any authentication credentials to the server */
if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) {
CLOSE_THE_SOCKET(sd);
return rc;
}
/* do whatever handshake is required */
if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) {
CLOSE_THE_SOCKET(sd);
return rc;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"sock_peer_try_connect: Connection across to server succeeded");
/* mark the connection as made */
pmix_globals.connected = true;
pmix_ptl_base_set_nonblocking(sd);
/* setup recv event */
event_assign(&pmix_client_globals.myserver.recv_event,
pmix_globals.evbase,
pmix_client_globals.myserver.sd,
EV_READ | EV_PERSIST,
pmix_ptl_base_recv_handler, &pmix_client_globals.myserver);
event_add(&pmix_client_globals.myserver.recv_event, 0);
pmix_client_globals.myserver.recv_ev_active = true;
/* setup send event */
event_assign(&pmix_client_globals.myserver.send_event,
pmix_globals.evbase,
pmix_client_globals.myserver.sd,
EV_WRITE|EV_PERSIST,
pmix_ptl_base_send_handler, &pmix_client_globals.myserver);
pmix_client_globals.myserver.send_ev_active = false;
return PMIX_SUCCESS;
}
static pmix_status_t send_recv(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_cbfunc_t cbfunc,
void *cbdata)
{
pmix_ptl_sr_t *ms;
pmix_output_verbose(5, pmix_globals.debug_output,
"[%s:%d] post send to server",
__FILE__, __LINE__);
ms = PMIX_NEW(pmix_ptl_sr_t);
ms->peer = peer;
ms->bfr = bfr;
ms->cbfunc = cbfunc;
ms->cbdata = cbdata;
event_assign(&ms->ev, pmix_globals.evbase, -1,
EV_WRITE, pmix_ptl_base_send_recv, ms);
event_active(&ms->ev, EV_WRITE, 1);
return PMIX_SUCCESS;
}
static pmix_status_t send_oneway(struct pmix_peer_t *peer,
pmix_buffer_t *bfr,
pmix_ptl_tag_t tag)
{
pmix_ptl_queue_t *q;
/* we have to transfer this to an event for thread
* safety as we need to post this message on the
* peer's send queue */
q = PMIX_NEW(pmix_ptl_queue_t);
q->peer = peer;
q->buf = bfr;
q->tag = tag;
event_assign(&q->ev, pmix_globals.evbase, -1,
EV_WRITE, pmix_ptl_base_send, q);
event_active(&q->ev, EV_WRITE, 1);
return PMIX_SUCCESS;
}
static pmix_status_t send_connect_ack(int sd)
{
char *msg;
pmix_usock_hdr_t hdr;
size_t sdsize=0, csize=0, len;
char *cred = NULL;
pmix_status_t rc;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: SEND CONNECT ACK");
/* setup the header */
memset(&hdr, 0, sizeof(pmix_usock_hdr_t));
hdr.pindex = -1;
hdr.tag = UINT32_MAX;
/* reserve space for the nspace and rank info */
sdsize = strlen(pmix_globals.myid.nspace) + 1 + sizeof(int);
/* get a credential, if the security system provides one. Not
* every SPC will do so, thus we must first check */
if (PMIX_SUCCESS != (rc = pmix_psec.create_cred(&pmix_client_globals.myserver,
PMIX_PROTOCOL_V1, &cred, &len))) {
return rc;
}
/* set the number of bytes to be read beyond the header */
hdr.nbytes = sdsize + strlen(PMIX_VERSION) + 1 + len; // must NULL terminate the VERSION string!
/* create a space for our message */
sdsize = (sizeof(hdr) + hdr.nbytes);
if (NULL == (msg = (char*)malloc(sdsize))) {
if (NULL != cred) {
free(cred);
}
return PMIX_ERR_OUT_OF_RESOURCE;
}
memset(msg, 0, sdsize);
/* load the message */
csize=0;
memcpy(msg, &hdr, sizeof(pmix_usock_hdr_t));
csize += sizeof(pmix_usock_hdr_t);
memcpy(msg+csize, pmix_globals.myid.nspace, strlen(pmix_globals.myid.nspace));
csize += strlen(pmix_globals.myid.nspace)+1;
memcpy(msg+csize, &pmix_globals.myid.rank, sizeof(int));
csize += sizeof(int);
memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
csize += strlen(PMIX_VERSION)+1;
if (NULL != cred) {
memcpy(msg+csize, cred, strlen(cred)); // leaves last position in msg set to NULL
}
if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(sd, msg, sdsize)) {
free(msg);
if (NULL != cred) {
free(cred);
}
return PMIX_ERR_UNREACH;
}
free(msg);
if (NULL != cred) {
free(cred);
}
return PMIX_SUCCESS;
}
/* we receive a connection acknowledgement from the server,
* consisting of nothing more than a status report. If success,
* then we initiate authentication method */
static pmix_status_t recv_connect_ack(int sd)
{
pmix_status_t reply;
pmix_status_t rc;
struct timeval tv, save;
pmix_socklen_t sz;
bool sockopt = true;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT ACK FROM SERVER");
/* get the current timeout value so we can reset to it */
sz = sizeof(save);
if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) {
if (ENOPROTOOPT == errno) {
sockopt = false;
} else {
return PMIX_ERR_UNREACH;
}
} else {
/* set a timeout on the blocking recv so we don't hang */
tv.tv_sec = 2;
tv.tv_usec = 0;
if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO");
return PMIX_ERR_UNREACH;
}
}
/* receive the status reply */
rc = pmix_ptl_base_recv_blocking(sd, (char*)&reply, sizeof(int));
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
return rc;
}
/* see if they want us to do the handshake */
if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
if (PMIX_SUCCESS != (rc = pmix_psec.client_handshake(&pmix_client_globals.myserver, sd))) {
return rc;
}
} else if (PMIX_SUCCESS != reply) {
return reply;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT CONFIRMATION");
/* receive our index into the server's client array */
rc = pmix_ptl_base_recv_blocking(sd, (char*)&pmix_globals.pindex, sizeof(int));
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
return rc;
}
if (sockopt) {
/* return the socket to normal */
if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
return PMIX_ERR_UNREACH;
}
}
return PMIX_SUCCESS;
}

47
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock.h Обычный файл
Просмотреть файл

@ -0,0 +1,47 @@
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef PMIX_PTL_USOCK_H
#define PMIX_PTL_USOCK_H
#include "src/mca/ptl/ptl.h"
BEGIN_C_DECLS
typedef struct {
pmix_ptl_base_component_t super;
char *tmpdir;
struct sockaddr_storage connection;
char *filename;
} pmix_ptl_usock_component_t;
/* header for messages */
typedef struct {
int pindex;
uint32_t tag;
size_t nbytes;
} pmix_usock_hdr_t;
extern pmix_ptl_usock_component_t mca_ptl_usock_component;
extern pmix_ptl_module_t pmix_ptl_usock_module;
END_C_DECLS
#endif /* PMIX_PTL_USOCK_H */

621
opal/mca/pmix/pmix2x/pmix/src/mca/ptl/usock/ptl_usock_component.c Обычный файл
Просмотреть файл

@ -0,0 +1,621 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennptlee and The University
* of Tennptlee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
* These symbols are in a file by themselves to provide nice linker
* semantics. Since linkers generally pull in symbols by object
* files, keeping these symbols as the only symbols in this file
* prevents utility programs such as "ompi_info" from having to import
* entire components just to query their version and parameters.
*/
#include <src/include/pmix_config.h>
#include <pmix_common.h>
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_UN_H
#include <sys/un.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#include <errno.h>
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/fd.h"
#include "src/util/show_help.h"
#include "src/util/strnlen.h"
#include "src/mca/psec/psec.h"
#include "src/server/pmix_server_ops.h"
#include "src/mca/ptl/base/base.h"
#include "src/mca/ptl/usock/ptl_usock.h"
static pmix_status_t component_open(void);
static pmix_status_t component_close(void);
static int component_query(pmix_mca_base_module_t **module, int *priority);
static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo,
bool *need_listener);
/*
* Instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
pmix_ptl_usock_component_t mca_ptl_usock_component = {
.super = {
.base = {
PMIX_PTL_BASE_VERSION_1_0_0,
/* Component name and version */
.pmix_mca_component_name = "usock",
PMIX_MCA_BASE_MAKE_VERSION(component,
PMIX_MAJOR_VERSION,
PMIX_MINOR_VERSION,
PMIX_RELEASE_VERSION),
/* Component open and close functions */
.pmix_mca_open_component = component_open,
.pmix_mca_close_component = component_close,
.pmix_mca_query_component = component_query
},
.priority = 5,
.uri = NULL,
.setup_listener = setup_listener
},
.tmpdir = NULL,
.filename = NULL
};
static void connection_handler(int sd, short args, void *cbdata);
static void listener_cb(int incoming_sd, void *cbdata);
static char *sec_mode = NULL;
pmix_status_t component_open(void)
{
char *tdir;
memset(&mca_ptl_usock_component.connection, 0, sizeof(mca_ptl_usock_component.connection));
/* check for environ-based directives
* on system tmpdir to use */
if (NULL == (tdir = getenv("PMIX_SYSTEM_TMPDIR"))) {
if (NULL == (tdir = getenv("TMPDIR"))) {
if (NULL == (tdir = getenv("TEMP"))) {
if (NULL == (tdir = getenv("TMP"))) {
tdir = "/tmp";
}
}
}
}
if (NULL != tdir) {
mca_ptl_usock_component.tmpdir = strdup(tdir);
}
return PMIX_SUCCESS;
}
pmix_status_t component_close(void)
{
if (NULL != sec_mode) {
free(sec_mode);
}
if (NULL != mca_ptl_usock_component.tmpdir) {
free(mca_ptl_usock_component.tmpdir);
}
if (NULL != mca_ptl_usock_component.super.uri) {
free(mca_ptl_usock_component.super.uri);
}
if (NULL != mca_ptl_usock_component.filename) {
/* remove the file */
unlink(mca_ptl_usock_component.filename);
free(mca_ptl_usock_component.filename);
}
return PMIX_SUCCESS;
}
static int component_query(pmix_mca_base_module_t **module, int *priority)
{
*module = (pmix_mca_base_module_t*)&pmix_ptl_usock_module;
return PMIX_SUCCESS;
}
/* if we are the server, then we need to setup a usock rendezvous
* point for legacy releases, but only do so if requested as some
* systems may not wish to support older releases. The system can,
* of course, simply use the MCA param method to disable this
* component (PMIX_MCA_ptl=^usock), or can tell us to disqualify
* ourselves using an info key to this API.
*
* NOTE: we accept MCA parameters, but info keys override them
*/
static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo,
bool *need_listener)
{
int flags;
size_t n;
pmix_listener_t *lt;
pmix_status_t rc;
socklen_t addrlen;
struct sockaddr_un *address;
bool disabled = false;
char *secmods, **options, *pmix_pid;
pid_t mypid;
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"ptl:usock setup_listener");
/* if we are not a server, then we shouldn't be doing this */
if (PMIX_PROC_SERVER != pmix_globals.proc_type) {
return PMIX_ERR_NOT_SUPPORTED;
}
/* scan the info keys and process any override instructions */
if (NULL != info) {
for (n=0; n < ninfo; n++) {
if (0 == strcmp(info[n].key, PMIX_USOCK_DISABLE)) {
if (PMIX_UNDEF == info[n].value.type) {
disabled = true;;
} else {
disabled = info[n].value.data.flag;
}
break;
}
}
}
/* see if we have been disabled */
if (disabled) {
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"ptl:usock not available");
return PMIX_ERR_NOT_AVAILABLE;
}
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"ptl:usock setting up listener");
addrlen = sizeof(struct sockaddr_un);
address = (struct sockaddr_un*)&mca_ptl_usock_component.connection;
address->sun_family = AF_UNIX;
/* any client we hear from will be using v1.x protocols. This
* means that they cannot tell us what security module they
* are using as this wasn't included in their handshake. So
* the best we can assume is that they are using the highest
* priority default we have */
secmods = pmix_psec.get_available_modules();
options = pmix_argv_split(secmods, ',');
sec_mode = strdup(options[0]);
pmix_argv_free(options);
free(secmods);
/* define the listener */
lt = PMIX_NEW(pmix_listener_t);
/* for now, just setup the v1.1 series rendezvous point
* we use the pid to reduce collisions */
mypid = getpid();
if (0 > asprintf(&pmix_pid, "%s/pmix-%d", mca_ptl_usock_component.tmpdir, mypid)) {
PMIX_RELEASE(lt);
return PMIX_ERR_NOMEM;
}
if ((strlen(pmix_pid) + 1) > sizeof(address->sun_path)-1) {
pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true,
mca_ptl_usock_component.tmpdir, pmix_pid);
free(pmix_pid);
PMIX_RELEASE(lt);
return PMIX_ERR_INVALID_LENGTH;
}
snprintf(address->sun_path, sizeof(address->sun_path)-1, "%s", pmix_pid);
free(pmix_pid);
/* set the URI */
lt->varname = strdup("PMIX_SERVER_URI");
if (0 > asprintf(&lt->uri, "%s:%lu:%s", pmix_globals.myid.nspace,
(unsigned long)pmix_globals.myid.rank, address->sun_path)) {
PMIX_RELEASE(lt);
return PMIX_ERR_NOMEM;
}
/* save the rendezvous filename for later removal */
mca_ptl_usock_component.filename = strdup(address->sun_path);
lt->protocol = PMIX_PROTOCOL_V1;
lt->ptl = (struct pmix_ptl_module_t*)&pmix_ptl_usock_module;
lt->cbfunc = connection_handler;
pmix_list_append(&pmix_ptl_globals.listeners, &lt->super);
/* create a listen socket for incoming connection attempts */
lt->socket = socket(PF_UNIX, SOCK_STREAM, 0);
if (lt->socket < 0) {
printf("%s:%d socket() failed\n", __FILE__, __LINE__);
goto sockerror;
}
/* Set the socket to close-on-exec so that no children inherit
* this FD */
if (pmix_fd_set_cloexec(lt->socket) != PMIX_SUCCESS) {
CLOSE_THE_SOCKET(lt->socket);
goto sockerror;
}
if (bind(lt->socket, (struct sockaddr*)address, addrlen) < 0) {
printf("%s:%d bind() failed\n", __FILE__, __LINE__);
CLOSE_THE_SOCKET(lt->socket);
goto sockerror;
}
/* chown as required */
if (lt->owner_given) {
if (0 != chown(address->sun_path, lt->owner, -1)) {
pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno));
CLOSE_THE_SOCKET(lt->socket);
goto sockerror;
}
}
if (lt->group_given) {
if (0 != chown(address->sun_path, -1, lt->group)) {
pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno));
CLOSE_THE_SOCKET(lt->socket);
goto sockerror;
}
}
/* set the mode as required */
if (0 != chmod(address->sun_path, lt->mode)) {
pmix_output(0, "CANNOT CHMOD socket %s: %s", address->sun_path, strerror (errno));
CLOSE_THE_SOCKET(lt->socket);
goto sockerror;
}
/* setup listen backlog to maximum allowed by kernel */
if (listen(lt->socket, SOMAXCONN) < 0) {
printf("%s:%d listen() failed\n", __FILE__, __LINE__);
CLOSE_THE_SOCKET(lt->socket);
goto sockerror;
}
/* set socket up to be non-blocking, otherwise accept could block */
if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) {
printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__);
CLOSE_THE_SOCKET(lt->socket);
goto sockerror;
}
flags |= O_NONBLOCK;
if (fcntl(lt->socket, F_SETFL, flags) < 0) {
printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__);
CLOSE_THE_SOCKET(lt->socket);
goto sockerror;
}
/* if the server will listen for us, then ask it to do so now */
rc = PMIX_ERR_NOT_SUPPORTED;
if (NULL != pmix_host_server.listener) {
rc = pmix_host_server.listener(lt->socket, listener_cb, (void*)lt);
}
if (PMIX_SUCCESS != rc) {
*need_listener = true;
}
return PMIX_SUCCESS;
sockerror:
pmix_list_remove_item(&pmix_ptl_globals.listeners, &lt->super);
PMIX_RELEASE(lt);
return PMIX_ERROR;
}
static void listener_cb(int incoming_sd, void *cbdata)
{
pmix_pending_connection_t *pending_connection;
/* throw it into our event library for processing */
pmix_output_verbose(8, pmix_ptl_base_framework.framework_output,
"listen_cb: pushing new connection %d into evbase",
incoming_sd);
pending_connection = PMIX_NEW(pmix_pending_connection_t);
pending_connection->sd = incoming_sd;
event_assign(&pending_connection->ev, pmix_globals.evbase, -1,
EV_WRITE, connection_handler, pending_connection);
event_active(&pending_connection->ev, EV_WRITE, 1);
}
/* Parse init-ack message:
* NSPACE<0><rank>VERSION<0>[CRED<0>]
*/
static pmix_status_t parse_connect_ack (char *msg, unsigned int len,
char **nspace, unsigned int *rank,
char **version, char **cred)
{
unsigned int msglen;
PMIX_STRNLEN(msglen, msg, len);
if (msglen < len) {
*nspace = msg;
msg += strlen(*nspace) + 1;
len -= strlen(*nspace) + 1;
} else {
return PMIX_ERR_BAD_PARAM;
}
PMIX_STRNLEN(msglen, msg, len);
if (msglen <= len) {
memcpy(rank, msg, sizeof(int));
msg += sizeof(int);
len -= sizeof(int);
} else {
return PMIX_ERR_BAD_PARAM;
}
PMIX_STRNLEN(msglen, msg, len);
if (msglen < len) {
*version = msg;
msg += strlen(*version) + 1;
len -= strlen(*version) + 1;
} else {
return PMIX_ERR_BAD_PARAM;
}
PMIX_STRNLEN(msglen, msg, len);
if (msglen < len)
*cred = msg;
else {
*cred = NULL;
}
return PMIX_SUCCESS;
}
static void connection_handler(int sd, short args, void *cbdata)
{
pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata;
char *msg, *nspace, *version, *cred;
pmix_status_t rc;
unsigned int rank;
pmix_usock_hdr_t hdr;
pmix_nspace_t *nptr, *tmp;
pmix_rank_info_t *info;
pmix_peer_t *psave = NULL;
bool found;
pmix_proc_t proc;
size_t len;
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"USOCK CONNECTION FROM PEER ON SOCKET %d", pnd->sd);
/* ensure all is zero'd */
memset(&hdr, 0, sizeof(pmix_usock_hdr_t));
/* get the header */
if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(pnd->sd, (char*)&hdr, sizeof(pmix_usock_hdr_t)))) {
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}
/* get the id, authentication and version payload (and possibly
* security credential) - to guard against potential attacks,
* we'll set an arbitrary limit per a define */
if (PMIX_MAX_CRED_SIZE < hdr.nbytes) {
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}
if (NULL == (msg = (char*)malloc(hdr.nbytes))) {
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}
if (PMIX_SUCCESS != pmix_ptl_base_recv_blocking(pnd->sd, msg, hdr.nbytes)) {
/* unable to complete the recv */
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"unable to complete recv of connect-ack with client ON SOCKET %d", pnd->sd);
free(msg);
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}
if (PMIX_SUCCESS != (rc = parse_connect_ack (msg, hdr.nbytes, &nspace,
&rank, &version, &cred))) {
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"error parsing connect-ack from client ON SOCKET %d", pnd->sd);
free(msg);
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"connect-ack recvd from peer %s:%d:%s on socket %d",
nspace, rank, version, pnd->sd);
/* do not check the version - we only retain it at this
* time in case we need to check it at some future date.
* For now, our intent is to retain backward compatibility
* and so we will assume that all versions are compatible. */
/* see if we know this nspace */
nptr = NULL;
PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_nspace_t) {
if (0 == strcmp(tmp->nspace, nspace)) {
nptr = tmp;
break;
}
}
if (NULL == nptr) {
/* we don't know this namespace, reject it */
free(msg);
/* send an error reply to the client */
rc = PMIX_ERR_NOT_FOUND;
goto error;
}
/* see if we have this peer in our list */
info = NULL;
found = false;
PMIX_LIST_FOREACH(info, &nptr->server->ranks, pmix_rank_info_t) {
if (info->rank == rank) {
found = true;
break;
}
}
if (!found) {
/* rank unknown, reject it */
free(msg);
/* send an error reply to the client */
rc = PMIX_ERR_NOT_FOUND;
goto error;
}
/* a peer can connect on multiple sockets since it can fork/exec
* a child that also calls PMIx_Init, so add it here if necessary.
* Create the tracker for this peer */
psave = PMIX_NEW(pmix_peer_t);
if (NULL == psave) {
rc = PMIX_ERR_NOMEM;
goto error;
}
PMIX_RETAIN(info);
psave->info = info;
info->proc_cnt++; /* increase number of processes on this rank */
psave->sd = pnd->sd;
if (0 > (psave->index = pmix_pointer_array_add(&pmix_server_globals.clients, psave))) {
free(msg);
info->proc_cnt--;
PMIX_RELEASE(info);
PMIX_RELEASE(psave);
/* probably cannot send an error reply if we are out of memory */
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}
/* get the appropriate compatibility modules */
if (PMIX_SUCCESS != pmix_psec.assign_module((struct pmix_peer_t*)psave, sec_mode)) {
info->proc_cnt--;
PMIX_RELEASE(info);
PMIX_RELEASE(psave);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
/* send an error reply to the client */
goto error;
}
/* the choice of PTL module was obviously made by the connecting
* tool as we received this request via that channel, so simply
* record it here for future use */
psave->compat.ptl = &pmix_ptl_usock_module;
/* validate the connection */
if (NULL == cred) {
len = 0;
} else {
len = strlen(cred);
}
if (PMIX_SUCCESS != (rc = pmix_psec.validate_connection((struct pmix_peer_t*)psave,
PMIX_PROTOCOL_V1, cred, len))) {
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"validation of client credentials failed: %s",
PMIx_Error_string(rc));
info->proc_cnt--;
PMIX_RELEASE(info);
PMIX_RELEASE(psave);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
/* send an error reply to the client */
goto error;
}
/* send them success */
rc = PMIX_SUCCESS;
if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
info->proc_cnt--;
PMIX_RELEASE(info);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}
/* send the client's array index */
if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
info->proc_cnt--;
PMIX_RELEASE(info);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"connect-ack from client completed");
/* let the host server know that this client has connected */
if (NULL != pmix_host_server.client_connected) {
(void)strncpy(proc.nspace, psave->info->nptr->nspace, PMIX_MAX_NSLEN);
proc.rank = psave->info->rank;
rc = pmix_host_server.client_connected(&proc, psave->info->server_object, NULL, NULL);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
info->proc_cnt--;
PMIX_RELEASE(info);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
CLOSE_THE_SOCKET(pnd->sd);
}
}
/* start the events for this client */
event_assign(&psave->recv_event, pmix_globals.evbase, pnd->sd,
EV_READ|EV_PERSIST, pmix_ptl_base_recv_handler, psave);
event_add(&psave->recv_event, NULL);
psave->recv_ev_active = true;
event_assign(&psave->send_event, pmix_globals.evbase, pnd->sd,
EV_WRITE|EV_PERSIST, pmix_ptl_base_send_handler, psave);
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"pmix:server client %s:%u has connected on socket %d",
psave->info->nptr->nspace, psave->info->rank, psave->sd);
PMIX_RELEASE(pnd);
return;
error:
/* send an error reply to the client */
if (PMIX_SUCCESS != pmix_ptl_base_send_blocking(pnd->sd, (char*)&rc, sizeof(int))) {
PMIX_ERROR_LOG(rc);
}
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
return;
}

14
opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_finalize.c
Просмотреть файл

@ -28,7 +28,6 @@
#include "src/class/pmix_object.h"
#include "src/client/pmix_client_ops.h"
#include "src/usock/usock.h"
#include "src/util/output.h"
#include "src/util/keyval_parse.h"
#include "src/util/show_help.h"
@ -37,6 +36,7 @@
#include "src/mca/pif/base/base.h"
#include "src/mca/pinstalldirs/base/base.h"
#include "src/mca/psec/base/base.h"
#include "src/mca/ptl/base/base.h"
#include "src/dstore/pmix_dstore.h"
#include PMIX_EVENT_HEADER
@ -73,17 +73,7 @@ void pmix_rte_finalize(void)
}
/* cleanup communications */
pmix_usock_finalize();
if (!pmix_globals.external_evbase) {
/* finalize the progress thread */
(void)pmix_progress_thread_finalize(NULL);
}
if (PMIX_PROC_SERVER != pmix_globals.proc_type &&
0 <= pmix_client_globals.myserver.sd) {
CLOSE_THE_SOCKET(pmix_client_globals.myserver.sd);
}
(void)pmix_mca_base_framework_close(&pmix_ptl_base_framework);
#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
pmix_dstore_finalize();
#endif /* PMIX_ENABLE_DSTORE */

68
opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_init.c
Просмотреть файл

@ -44,12 +44,13 @@
#include "src/mca/pif/base/base.h"
#include "src/mca/pinstalldirs/base/base.h"
#include "src/mca/psec/base/base.h"
#include "src/mca/ptl/base/base.h"
#include "src/event/pmix_event.h"
#include "src/include/types.h"
#include "src/usock/usock.h"
#include "src/util/error.h"
#include "src/util/keyval_parse.h"
#include "src/buffer_ops/buffer_ops.h"
#include "src/runtime/pmix_rte.h"
#include "src/runtime/pmix_progress_threads.h"
@ -82,11 +83,10 @@ PMIX_EXPORT pmix_globals_t pmix_globals = {
int pmix_rte_init(pmix_proc_type_t type,
pmix_info_t info[], size_t ninfo,
pmix_usock_cbfunc_t notifycbfunc)
pmix_ptl_cbfunc_t notifycbfunc)
{
int ret, debug_level;
char *error = NULL, *evar;
char *param;
size_t n;
if( ++pmix_initialized != 1 ) {
@ -179,19 +179,21 @@ int pmix_rte_init(pmix_proc_type_t type,
}
pmix_bfrop_open();
#if 0
/* open the bfrops - we will select the active plugin later */
if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_bfrops_base_framework, 0)) ) {
error = "pmix_bfrops_base_open";
goto return_error;
}
if( PMIX_SUCCESS != (ret = pmix_bfrop_base_select()) ) {
error = "pmix_bfrops_base_select";
goto return_error;
}
#endif
/* the choice of modules to use when communicating with a peer
* will be done by the individual init functions and at the
* time of connection to that peer */
/* open the psec and select the default module for this environment */
/* open the ptl and select the active plugins */
if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) {
error = "pmix_ptl_base_open";
goto return_error;
}
if( PMIX_SUCCESS != (ret = pmix_ptl_base_select()) ) {
error = "pmix_ptl_base_select";
goto return_error;
}
/* open the psec and select the active plugins */
if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_psec_base_framework, 0))) {
error = "pmix_psec_base_open";
goto return_error;
@ -200,13 +202,6 @@ int pmix_rte_init(pmix_proc_type_t type,
error = "pmix_psec_base_select";
goto return_error;
}
param = getenv("PMIX_SEC_MODULE"); // if directive was given, use it
pmix_globals.mypeer->compat.psec = pmix_psec_base_assign_module(param);
if (NULL == pmix_globals.mypeer->compat.psec) {
ret = PMIX_ERR_NOT_SUPPORTED;
error = "no psec module";
goto return_error;
}
/* initialize pif framework */
if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_pif_base_framework, 0))) {
@ -216,6 +211,8 @@ int pmix_rte_init(pmix_proc_type_t type,
/* tell libevent that we need thread support */
pmix_event_use_threads();
/* if an external event base wasn't provide, create one */
if (!pmix_globals.external_evbase) {
/* create an event base and progress thread for us */
if (NULL == (pmix_globals.evbase = pmix_progress_thread_init(NULL))) {
@ -225,33 +222,6 @@ int pmix_rte_init(pmix_proc_type_t type,
}
}
#if 0
/* open the PMIx Transport Layer (ptl) - we will select the
* active plugin later */
if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) {
error = "pmix_ptl_base_open";
goto return_error;
}
if( PMIX_SUCCESS != (ret = pmix_ptl_base_select()) ) {
error = "pmix_ptl_base_select";
goto return_error;
}
/* open the dstor framework - we will select the active
* plugin later */
if( PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_pdstor_base_framework, 0)) ) {
error = "pmix_pdstor_base_open";
goto return_error;
}
if( PMIX_SUCCESS != (ret = pmix_pdstor_base_select()) ) {
error = "pmix_pdstor_base_select";
goto return_error;
}
#endif
/* setup the usock service */
pmix_usock_init(notifycbfunc);
return PMIX_SUCCESS;
return_error:

3
opal/mca/pmix/pmix2x/pmix/src/runtime/pmix_rte.h
Просмотреть файл

@ -34,6 +34,7 @@
#include PMIX_EVENT_HEADER
#include "src/include/pmix_globals.h"
#include "src/mca/ptl/ptl_types.h"
BEGIN_C_DECLS
@ -58,7 +59,7 @@ extern const char pmix_version_string[];
*/
pmix_status_t pmix_rte_init(pmix_proc_type_t type,
pmix_info_t info[], size_t ninfo,
pmix_usock_cbfunc_t cbfunc);
pmix_ptl_cbfunc_t cbfunc);
/**
* Finalize the PMIX layer, including the MCA system.

3
opal/mca/pmix/pmix2x/pmix/src/server/Makefile.include
Просмотреть файл

@ -20,5 +20,4 @@ sources += \
server/pmix_server.c \
server/pmix_server_ops.c \
server/pmix_server_regex.c \
server/pmix_server_get.c \
server/pmix_server_listener.c
server/pmix_server_get.c

328
opal/mca/pmix/pmix2x/pmix/src/server/pmix_server.c
Просмотреть файл

@ -61,7 +61,7 @@
#include "src/mca/pinstalldirs/base/base.h"
#include "src/runtime/pmix_progress_threads.h"
#include "src/runtime/pmix_rte.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/base/base.h"
#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
#include "src/dstore/pmix_dstore.h"
#endif /* PMIX_ENABLE_DSTORE */
@ -74,60 +74,19 @@ pmix_server_globals_t pmix_server_globals = {{{0}}};
// local variables
static char *security_mode = NULL;
static char *ptl_mode = NULL;
static pid_t mypid;
static char *mytmpdir = NULL;
static char *systmpdir = NULL;
// local functions for connection support
static void server_message_handler(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void server_message_handler(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata);
static inline int _my_client(const char *nspace, pmix_rank_t rank);
/* queue a message to be sent to one of our procs - must
* provide the following params:
*
* p - the peer object of the process
* t - tag to be sent to
* b - buffer to be sent
*/
void pmix_server_queue_message(int fd, short args, void *cbdata)
{
pmix_usock_queue_t *queue = (pmix_usock_queue_t*)cbdata;
pmix_usock_send_t *snd;
pmix_output_verbose(2, pmix_globals.debug_output,
"[%s:%d] queue callback called: reply to %s:%d on tag %d",
__FILE__, __LINE__,
(queue->peer)->info->nptr->nspace,
(queue->peer)->info->rank, (queue->tag));
snd = PMIX_NEW(pmix_usock_send_t);
snd->hdr.pindex = pmix_globals.pindex;
snd->hdr.tag = (queue->tag);
snd->hdr.nbytes = (queue->buf)->bytes_used;
snd->data = (queue->buf);
/* always start with the header */
snd->sdptr = (char*)&snd->hdr;
snd->sdbytes = sizeof(pmix_usock_hdr_t);
/* if there is no message on-deck, put this one there */
if (NULL == (queue->peer)->send_msg) {
(queue->peer)->send_msg = snd;
} else {
/* add it to the queue */
pmix_list_append(&(queue->peer)->send_queue, &snd->super);
}
/* ensure the send event is active */
if (!(queue->peer)->send_ev_active) {
event_add(&(queue->peer)->send_event, 0);
(queue->peer)->send_ev_active = true;
}
PMIX_RELEASE(queue);
}
static pmix_status_t initialize_server_base(pmix_server_module_t *module)
{
char *tdir, *evar;
char * pmix_pid;
pmix_listener_t *listener;
char *evar;
/* look for our namespace, if one was given */
if (NULL == (evar = getenv("PMIX_SERVER_NAMESPACE"))) {
@ -154,7 +113,6 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module)
PMIX_CONSTRUCT(&pmix_server_globals.events, pmix_list_t);
PMIX_CONSTRUCT(&pmix_server_globals.local_reqs, pmix_list_t);
PMIX_CONSTRUCT(&pmix_server_globals.notifications, pmix_ring_buffer_t);
PMIX_CONSTRUCT(&pmix_server_globals.listeners, pmix_list_t);
pmix_ring_buffer_init(&pmix_server_globals.notifications, 256);
pmix_output_verbose(2, pmix_globals.debug_output,
@ -164,65 +122,25 @@ static pmix_status_t initialize_server_base(pmix_server_module_t *module)
memset(&pmix_host_server, 0, sizeof(pmix_server_module_t));
pmix_host_server = *module;
/* find the temp dir */
if (NULL != mytmpdir) {
tdir = mytmpdir;
} else if (NULL == (tdir = getenv("TMPDIR"))) {
if (NULL == (tdir = getenv("TEMP"))) {
if (NULL == (tdir = getenv("TMP"))) {
tdir = "/tmp";
}
}
}
/* for now, just setup the v1.1 series rendezvous point
* we use the pid to reduce collisions */
if (0 > asprintf(&pmix_pid, "%s/pmix-%d", tdir, mypid)) {
return PMIX_ERR_NOMEM;
}
if ((strlen(pmix_pid) + 1) > sizeof(listener->address.sun_path)-1) {
pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true, tdir, pmix_pid);
free(pmix_pid);
return PMIX_ERR_INVALID_LENGTH;
}
listener = PMIX_NEW(pmix_listener_t);
snprintf(listener->address.sun_path, sizeof(listener->address.sun_path)-1, "%s", pmix_pid);
if (0 > asprintf(&listener->uri, "%s:%lu:%s", pmix_globals.myid.nspace,
(unsigned long)pmix_globals.myid.rank, listener->address.sun_path)) {
free(pmix_pid);
return PMIX_ERR_NOMEM;
}
listener->varname = strdup("PMIX_SERVER_URI");
listener->protocol = PMIX_PROTOCOL_V1;
pmix_list_append(&pmix_server_globals.listeners, &listener->super);
free(pmix_pid);
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:server constructed uri %s", listener->uri);
return PMIX_SUCCESS;
}
PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
pmix_info_t info[], size_t ninfo)
{
pmix_usock_posted_recv_t *req;
pmix_ptl_posted_recv_t *req;
pmix_status_t rc;
size_t n, m;
pmix_kval_t kv;
pmix_listener_t *lt;
bool need_listener;
int myhostnamelen = 30;
char myhostname[myhostnamelen];
char *pmix_pid, *tdir;
char **protected = NULL;
bool protect;
bool tool_support = false;
bool system_tool = false;
bool session_tool = false;
pmix_listener_t *tl;
char *protected[] = {
PMIX_USERID,
PMIX_GRPID,
PMIX_SOCKET_MODE,
PMIX_SERVER_TOOL_SUPPORT,
PMIX_SERVER_SYSTEM_SUPPORT,
NULL
};
if (0 < pmix_globals.init_cntr) {
return PMIX_SUCCESS;
@ -238,26 +156,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
return rc;
}
/* Check for the info keys that are not independent from
* initialize_server_base() and even may be needed there */
if (NULL != info) {
for (n=0; n < ninfo; n++) {
if (0 == strcmp(info[n].key, PMIX_SERVER_TMPDIR) &&
NULL == mytmpdir) {
mytmpdir = strdup(info[n].value.data.string);
/* push this onto our protected list of keys not
* to be passed to the clients */
pmix_argv_append_nosize(&protected, PMIX_SERVER_TMPDIR);
} else if (0 == strcmp(info[n].key, PMIX_SYSTEM_TMPDIR) &&
NULL == systmpdir) {
systmpdir = strdup(info[n].value.data.string);
/* push this onto our protected list of keys not
* to be passed to the clients */
pmix_argv_append_nosize(&protected, PMIX_SYSTEM_TMPDIR);
}
}
}
if (0 != (rc = initialize_server_base(module))) {
return rc;
}
@ -268,149 +166,17 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
}
#endif /* PMIX_ENABLE_DSTORE */
/* check the info keys for a directive about the uid/gid
* to be set for the rendezvous file */
if (NULL != info) {
for (n=0; n < ninfo; n++) {
if (0 == strcmp(info[n].key, PMIX_USERID)) {
/* the userid is in the uint32_t storage */
PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
lt->owner = info[n].value.data.uint32;
lt->owner_given = true;
}
/* push this onto our protected list of keys not
* to be passed to the clients */
pmix_argv_append_nosize(&protected, PMIX_USERID);
} else if (0 == strcmp(info[n].key, PMIX_GRPID)) {
/* the grpid is in the uint32_t storage */
PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
lt->group = info[n].value.data.uint32;
lt->group_given = true;
}
/* push this onto our protected list of keys not
* to be passed to the clients */
pmix_argv_append_nosize(&protected, PMIX_GRPID);
} else if (0 == strcmp(info[n].key, PMIX_SOCKET_MODE)) {
/* socket mode is in the uint32_t storage */
PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
lt->mode = info[n].value.data.uint32;
}
} else if (0 == strcmp(info[n].key, PMIX_SERVER_TOOL_SUPPORT)) {
/* defer processing to ensure we pickup any tmpdir
* directives before setting location */
session_tool = true;
tool_support = true;
/* push this onto our protected list of keys not
* to be passed to the clients */
pmix_argv_append_nosize(&protected, PMIX_SERVER_TOOL_SUPPORT);
} else if (0 == strcmp(info[n].key, PMIX_SERVER_SYSTEM_SUPPORT)) {
/* we are also the system tool server */
system_tool = true;
tool_support = true;
/* push this onto our protected list of keys not
* to be passed to the clients */
pmix_argv_append_nosize(&protected, PMIX_SERVER_TOOL_SUPPORT);
}
}
}
if (tool_support) {
/* Get up to 30 chars of hostname.*/
gethostname(myhostname, myhostnamelen);
/* ensure it is NULL terminated */
myhostname[myhostnamelen-1] = '\0';
/* if we are to be the system tool, then we look for
* the system tmpdir and do not include a pid in
* the rendezvous point */
if (system_tool) {
if (NULL != systmpdir) {
tdir = systmpdir;
} else if (NULL == (tdir = getenv("TMPDIR"))) {
if (NULL == (tdir = getenv("TEMP"))) {
if (NULL == (tdir = getenv("TMP"))) {
tdir = "/tmp";
}
}
}
if (0 > asprintf(&pmix_pid, "%s/pmix.sys.%s", tdir, myhostname)) {
return PMIX_ERR_NOMEM;
}
if ((strlen(pmix_pid) + 1) > sizeof(tl->address.sun_path)-1) {
pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true, tdir, pmix_pid);
free(pmix_pid);
return PMIX_ERR_INVALID_LENGTH;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"%s:%d dropping system tool at %s",
pmix_globals.myid.nspace,
pmix_globals.myid.rank, pmix_pid);
/* create the listener for this point */
tl = PMIX_NEW(pmix_listener_t);
tl -> address.sun_family = AF_UNIX;
tl->protocol = PMIX_PROTOCOL_TOOL;
snprintf(tl->address.sun_path, sizeof(tl->address.sun_path) - 1, "%s", pmix_pid);
free(pmix_pid);
pmix_list_append(&pmix_server_globals.listeners, &tl->super);
}
/* if we are a session tool, then we use the session tmpdir
* and append our pid */
if (session_tool) {
if (NULL != mytmpdir) {
tdir = mytmpdir;
} else if (NULL == (tdir = getenv("TMPDIR"))) {
if (NULL == (tdir = getenv("TEMP"))) {
if (NULL == (tdir = getenv("TMP"))) {
tdir = "/tmp";
}
}
}
/* mark this with my pid */
if (0 > asprintf(&pmix_pid, "%s/pmix.%s.tool.%d", tdir, myhostname, mypid)) {
return PMIX_ERR_NOMEM;
}
if ((strlen(pmix_pid) + 1) > sizeof(tl->address.sun_path)-1) {
pmix_show_help("help-pmix-server.txt", "rnd-path-too-long", true, tdir, pmix_pid);
free(pmix_pid);
return PMIX_ERR_INVALID_LENGTH;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"%s:%d dropping session tool at %s",
pmix_globals.myid.nspace,
pmix_globals.myid.rank, pmix_pid);
/* create the listener for this point */
tl = PMIX_NEW(pmix_listener_t);
tl -> address.sun_family = AF_UNIX;
tl->protocol = PMIX_PROTOCOL_TOOL;
snprintf(tl->address.sun_path, sizeof(tl->address.sun_path) - 1, "%s", pmix_pid);
free(pmix_pid);
pmix_list_append(&pmix_server_globals.listeners, &tl->super);
}
/* we don't provide a URI for this listener as we don't pass
* the TOOL connection URI to a child process */
pmix_server_globals.tool_connections_allowed = true;
}
/* setup the wildcard recv for inbound messages from clients */
req = PMIX_NEW(pmix_usock_posted_recv_t);
req = PMIX_NEW(pmix_ptl_posted_recv_t);
req->tag = UINT32_MAX;
req->cbfunc = server_message_handler;
/* add it to the end of the list of recvs */
pmix_list_append(&pmix_usock_globals.posted_recvs, &req->super);
pmix_list_append(&pmix_ptl_globals.posted_recvs, &req->super);
/* start listening */
need_listener = false;
PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
if (PMIX_SUCCESS != pmix_prepare_listening(lt, &need_listener)) {
pmix_show_help("help-pmix-server.txt", "listener-failed-start", true, lt->address.sun_path);
PMIx_server_finalize();
return PMIX_ERR_INIT;
}
}
if (need_listener) {
if (PMIX_SUCCESS != pmix_start_listening()) {
pmix_show_help("help-pmix-server.txt", "listener-thread-start", true);
PMIx_server_finalize();
return PMIX_ERR_INIT;
}
if (PMIX_SUCCESS != pmix_ptl_base_start_listening(info, ninfo)) {
pmix_show_help("help-pmix-server.txt", "listener-thread-start", true);
PMIx_server_finalize();
return PMIX_ERR_INIT;
}
/* check the info keys for info we
@ -420,12 +186,10 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
for (n=0; n < ninfo; n++) {
/* check the list of protected keys */
protect = false;
if (NULL != protected) {
for (m=0; NULL != protected[m]; m++) {
if (0 == strcmp(info[n].key, protected[m])) {
protect = true;
break;
}
for (m=0; NULL != protected[m]; m++) {
if (0 == strcmp(info[n].key, protected[m])) {
protect = true;
break;
}
}
if (protect) {
@ -451,6 +215,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module,
PMIX_DESTRUCT(&kv);
}
/* get our available security modules */
security_mode = pmix_psec.get_available_modules();
/* get our available ptl modules */
ptl_mode = pmix_ptl.get_available_modules();
++pmix_globals.init_cntr;
return PMIX_SUCCESS;
@ -471,14 +241,13 @@ static void cleanup_server_state(void)
PMIX_LIST_DESTRUCT(&pmix_server_globals.remote_pnd);
PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs);
PMIX_DESTRUCT(&pmix_server_globals.gdata);
PMIX_LIST_DESTRUCT(&pmix_server_globals.listeners);
if (NULL != mytmpdir) {
free(mytmpdir);
if (NULL != security_mode) {
free(security_mode);
}
if (NULL != systmpdir) {
free(systmpdir);
if (NULL != ptl_mode) {
free(ptl_mode);
}
pmix_bfrop_close();
@ -496,9 +265,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:server finalize called");
if (pmix_server_globals.listen_thread_active) {
pmix_stop_listening();
}
pmix_ptl_base_stop_listening();
cleanup_server_state();
pmix_output_verbose(2, pmix_globals.debug_output,
@ -1081,13 +848,15 @@ PMIX_EXPORT pmix_status_t PMIx_server_setup_fork(const pmix_proc_t *proc, char *
(void)snprintf(rankstr, 127, "%d", proc->rank);
pmix_setenv("PMIX_RANK", rankstr, true, env);
/* pass our rendezvous info */
PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
PMIX_LIST_FOREACH(lt, &pmix_ptl_globals.listeners, pmix_listener_t) {
if (NULL != lt->uri && NULL != lt->varname) {
pmix_setenv(lt->varname, lt->uri, true, env);
}
}
/* pass our active security mode */
/* pass our active security modules */
pmix_setenv("PMIX_SECURITY_MODE", security_mode, true, env);
/* pass our available ptl modules */
pmix_setenv("PMIX_PTL_MODULE", ptl_mode, true, env);
#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
/* pass dstore path to files */
@ -1661,12 +1430,17 @@ static void op_cbfunc(pmix_status_t status, void *cbdata)
if (PMIX_SUCCESS != (rc = pmix_bfrop.pack(reply, &status, 1, PMIX_STATUS))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(reply);
PMIX_RELEASE(cd);
return;
}
/* the function that created the server_caddy did a
* retain on the peer, so we don't have to worry about
* it still being present - send a copy to the originator */
PMIX_SERVER_QUEUE_REPLY(cd->peer, cd->hdr.tag, reply);
if (PMIX_SUCCESS != (rc = pmix_ptl.send_oneway(cd->peer, reply, cd->hdr.tag))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(reply);
}
/* cleanup */
PMIX_RELEASE(cd);
}
@ -2237,7 +2011,7 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag,
pmix_bfrop.copy_payload(reply, &(pmix_server_globals.gdata));
#endif
PMIX_SERVER_QUEUE_REPLY(peer, tag, reply);
return PMIX_SUCCESS;
return PMIX_SUCCESS; // don't reply twice
}
if (PMIX_ABORT_CMD == cmd) {
@ -2250,7 +2024,10 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag,
if (PMIX_COMMIT_CMD == cmd) {
rc = pmix_server_commit(peer, buf);
return rc;
reply = PMIX_NEW(pmix_buffer_t);
pmix_bfrop.pack(reply, &rc, 1, PMIX_STATUS);
PMIX_SERVER_QUEUE_REPLY(peer, tag, reply);
return PMIX_SUCCESS; // don't reply twice
}
if (PMIX_FENCENB_CMD == cmd) {
@ -2390,7 +2167,8 @@ static pmix_status_t server_switchyard(pmix_peer_t *peer, uint32_t tag,
return PMIX_ERR_NOT_SUPPORTED;
}
static void server_message_handler(struct pmix_peer_t *pr, pmix_usock_hdr_t *hdr,
static void server_message_handler(struct pmix_peer_t *pr,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_peer_t *peer = (pmix_peer_t*)pr;

1
opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_get.c
Просмотреть файл

@ -52,7 +52,6 @@
#include "src/util/error.h"
#include "src/util/output.h"
#include "src/util/pmix_environ.h"
#include "src/usock/usock.h"
#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
#include "src/dstore/pmix_dstore.h"
#endif /* PMIX_ENABLE_DSTORE */

972
opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_listener.c
Просмотреть файл

@ -1,972 +0,0 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
* All rights reserved.
* Copyright (c) 2016 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <src/include/pmix_config.h>
#include <src/include/types.h>
#include <src/include/pmix_stdint.h>
#include <src/include/pmix_socket_errno.h>
#include <pmix_server.h>
#include "src/include/pmix_globals.h"
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#include <fcntl.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_UN_H
#include <sys/un.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include <ctype.h>
#include <sys/stat.h>
#include PMIX_EVENT_HEADER
#include <pthread.h>
#include "src/class/pmix_list.h"
#include "src/buffer_ops/buffer_ops.h"
#include "src/util/argv.h"
#include "src/util/error.h"
#include "src/util/fd.h"
#include "src/util/getid.h"
#include "src/util/output.h"
#include "src/util/pmix_environ.h"
#include "src/util/strnlen.h"
#include "src/usock/usock.h"
#include "src/mca/psec/psec.h"
#include "pmix_server_ops.h"
// local functions for connection support
static void* listen_thread(void *obj);
static void listener_cb(int incoming_sd, void *cbdata);
static void connection_handler(int incoming_sd, short flags, void* cbdata);
static void tool_handler(int incoming_sd, short flags, void* cbdata);
static char *myversion = NULL;
static pthread_t engine;
/*
* start listening on our rendezvous file
*/
pmix_status_t pmix_prepare_listening(pmix_listener_t *lt, bool *need_listener)
{
int flags;
pmix_status_t rc;
socklen_t addrlen;
char *ptr;
struct sockaddr_un *address = &lt->address;
/* create a listen socket for incoming connection attempts */
lt->socket = socket(PF_UNIX, SOCK_STREAM, 0);
if (lt->socket < 0) {
printf("%s:%d socket() failed\n", __FILE__, __LINE__);
return PMIX_ERROR;
}
/* Set the socket to close-on-exec so that no children inherit
* this FD */
if (pmix_fd_set_cloexec(lt->socket) != PMIX_SUCCESS) {
CLOSE_THE_SOCKET(lt->socket);
return PMIX_ERROR;
}
addrlen = sizeof(struct sockaddr_un);
if (bind(lt->socket, (struct sockaddr*)address, addrlen) < 0) {
printf("%s:%d bind() failed\n", __FILE__, __LINE__);
return PMIX_ERROR;
}
/* chown as required */
if (lt->owner_given) {
if (0 != chown(address->sun_path, lt->owner, -1)) {
pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno));
goto sockerror;
}
}
if (lt->group_given) {
if (0 != chown(address->sun_path, -1, lt->group)) {
pmix_output(0, "CANNOT CHOWN socket %s: %s", address->sun_path, strerror (errno));
goto sockerror;
}
}
/* set the mode as required */
if (0 != chmod(address->sun_path, lt->mode)) {
pmix_output(0, "CANNOT CHMOD socket %s: %s", address->sun_path, strerror (errno));
goto sockerror;
}
/* setup listen backlog to maximum allowed by kernel */
if (listen(lt->socket, SOMAXCONN) < 0) {
printf("%s:%d listen() failed\n", __FILE__, __LINE__);
goto sockerror;
}
/* set socket up to be non-blocking, otherwise accept could block */
if ((flags = fcntl(lt->socket, F_GETFL, 0)) < 0) {
printf("%s:%d fcntl(F_GETFL) failed\n", __FILE__, __LINE__);
goto sockerror;
}
flags |= O_NONBLOCK;
if (fcntl(lt->socket, F_SETFL, flags) < 0) {
printf("%s:%d fcntl(F_SETFL) failed\n", __FILE__, __LINE__);
goto sockerror;
}
if (NULL == myversion) {
/* setup my version for validating connections - we
* only check the major version numbers */
myversion = strdup(PMIX_VERSION);
/* find the first '.' */
ptr = strchr(myversion, '.');
if (NULL != ptr) {
++ptr;
/* stop it at the second '.', if present */
if (NULL != (ptr = strchr(ptr, '.'))) {
*ptr = '\0';
}
}
}
/* if the server will listen for us, then ask it to do so now */
rc = PMIX_ERR_NOT_SUPPORTED;
if (NULL != pmix_host_server.listener) {
rc = pmix_host_server.listener(lt->socket, listener_cb, (void*)lt);
}
if (PMIX_SUCCESS != rc && !pmix_server_globals.listen_thread_active) {
*need_listener = true;
}
return PMIX_SUCCESS;
sockerror:
(void)close(lt->socket);
lt->socket = -1;
return PMIX_ERROR;
}
pmix_status_t pmix_start_listening(void) {
/*** spawn internal listener thread */
if (0 > pipe(pmix_server_globals.stop_thread)) {
PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO);
return PMIX_ERR_OUT_OF_RESOURCE;
}
/* Make sure the pipe FDs are set to close-on-exec so that
they don't leak into children */
if (pmix_fd_set_cloexec(pmix_server_globals.stop_thread[0]) != PMIX_SUCCESS ||
pmix_fd_set_cloexec(pmix_server_globals.stop_thread[1]) != PMIX_SUCCESS) {
PMIX_ERROR_LOG(PMIX_ERR_IN_ERRNO);
close(pmix_server_globals.stop_thread[0]);
close(pmix_server_globals.stop_thread[1]);
return PMIX_ERR_OUT_OF_RESOURCE;
}
/* fork off the listener thread */
pmix_server_globals.listen_thread_active = true;
if (0 > pthread_create(&engine, NULL, listen_thread, NULL)) {
pmix_server_globals.listen_thread_active = false;
return PMIX_ERROR;
}
return PMIX_SUCCESS;
}
void pmix_stop_listening(void)
{
int i;
pmix_listener_t *lt;
pmix_output_verbose(8, pmix_globals.debug_output,
"listen_thread: shutdown");
if (!pmix_server_globals.listen_thread_active) {
/* nothing we can do */
return;
}
/* mark it as inactive */
pmix_server_globals.listen_thread_active = false;
/* use the block to break it loose just in
* case the thread is blocked in a call to select for
* a long time */
i=1;
if (0 > write(pmix_server_globals.stop_thread[1], &i, sizeof(int))) {
return;
}
/* wait for thread to exit */
pthread_join(engine, NULL);
/* close the sockets to remove the connection points */
PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
CLOSE_THE_SOCKET(lt->socket);
lt->socket = -1;
}
}
static void* listen_thread(void *obj)
{
int rc, max, accepted_connections;
socklen_t addrlen = sizeof(struct sockaddr_storage);
pmix_pending_connection_t *pending_connection;
struct timeval timeout;
fd_set readfds;
pmix_listener_t *lt;
pmix_output_verbose(8, pmix_globals.debug_output,
"listen_thread: active");
while (pmix_server_globals.listen_thread_active) {
FD_ZERO(&readfds);
max = -1;
PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
FD_SET(lt->socket, &readfds);
max = (lt->socket > max) ? lt->socket : max;
}
/* add the stop_thread fd */
FD_SET(pmix_server_globals.stop_thread[0], &readfds);
max = (pmix_server_globals.stop_thread[0] > max) ? pmix_server_globals.stop_thread[0] : max;
/* set timeout interval */
timeout.tv_sec = 2;
timeout.tv_usec = 0;
/* Block in a select to avoid hammering the cpu. If a connection
* comes in, we'll get woken up right away.
*/
rc = select(max + 1, &readfds, NULL, NULL, &timeout);
if (!pmix_server_globals.listen_thread_active) {
/* we've been asked to terminate */
close(pmix_server_globals.stop_thread[0]);
close(pmix_server_globals.stop_thread[1]);
return NULL;
}
if (rc < 0) {
continue;
}
/* Spin accepting connections until all active listen sockets
* do not have any incoming connections, pushing each connection
* onto the event queue for processing
*/
do {
accepted_connections = 0;
PMIX_LIST_FOREACH(lt, &pmix_server_globals.listeners, pmix_listener_t) {
/* according to the man pages, select replaces the given descriptor
* set with a subset consisting of those descriptors that are ready
* for the specified operation - in this case, a read. So we need to
* first check to see if this file descriptor is included in the
* returned subset
*/
if (0 == FD_ISSET(lt->socket, &readfds)) {
/* this descriptor is not included */
continue;
}
/* this descriptor is ready to be read, which means a connection
* request has been received - so harvest it. All we want to do
* here is accept the connection and push the info onto the event
* library for subsequent processing - we don't want to actually
* process the connection here as it takes too long, and so the
* OS might start rejecting connections due to timeout.
*/
pending_connection = PMIX_NEW(pmix_pending_connection_t);
pending_connection->protocol = lt->protocol;
if (PMIX_PROTOCOL_TOOL == lt->protocol) {
event_assign(&pending_connection->ev, pmix_globals.evbase, -1,
EV_WRITE, tool_handler, pending_connection);
} else {
event_assign(&pending_connection->ev, pmix_globals.evbase, -1,
EV_WRITE, connection_handler, pending_connection);
}
pending_connection->sd = accept(lt->socket,
(struct sockaddr*)&(pending_connection->addr),
&addrlen);
if (pending_connection->sd < 0) {
PMIX_RELEASE(pending_connection);
if (pmix_socket_errno != EAGAIN ||
pmix_socket_errno != EWOULDBLOCK) {
if (EMFILE == pmix_socket_errno ||
ENOBUFS == pmix_socket_errno ||
ENOMEM == pmix_socket_errno) {
PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE);
} else if (EINVAL == pmix_socket_errno ||
EINTR == pmix_socket_errno) {
/* race condition at finalize */
goto done;
} else if (ECONNABORTED == pmix_socket_errno) {
/* they aborted the attempt */
continue;
} else {
pmix_output(0, "listen_thread: accept() failed: %s (%d).",
strerror(pmix_socket_errno), pmix_socket_errno);
}
goto done;
}
continue;
}
pmix_output_verbose(8, pmix_globals.debug_output,
"listen_thread: new connection: (%d, %d)",
pending_connection->sd, pmix_socket_errno);
/* activate the event */
event_active(&pending_connection->ev, EV_WRITE, 1);
accepted_connections++;
}
} while (accepted_connections > 0);
}
done:
pmix_server_globals.listen_thread_active = false;
return NULL;
}
static void listener_cb(int incoming_sd, void *cbdata)
{
pmix_pending_connection_t *pending_connection;
pmix_listener_t *lt = (pmix_listener_t*)cbdata;
/* throw it into our event library for processing */
pmix_output_verbose(8, pmix_globals.debug_output,
"listen_cb: pushing new connection %d into evbase",
incoming_sd);
pending_connection = PMIX_NEW(pmix_pending_connection_t);
pending_connection->sd = incoming_sd;
pending_connection->protocol = lt->protocol;
event_assign(&pending_connection->ev, pmix_globals.evbase, -1,
EV_WRITE, connection_handler, pending_connection);
event_active(&pending_connection->ev, EV_WRITE, 1);
}
/* process the callback with tool connection info */
static void process_cbfunc(int sd, short args, void *cbdata)
{
pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata;
pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cd->cbdata;
pmix_nspace_t *nptr;
pmix_rank_info_t *info;
int rc;
/* send this status as well so they don't hang */
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&cd->status, sizeof(pmix_status_t)))) {
PMIX_ERROR_LOG(rc);
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
PMIX_RELEASE(cd);
return;
}
/* if the request failed, then we are done */
if (PMIX_SUCCESS != cd->status) {
PMIX_RELEASE(pnd);
PMIX_RELEASE(cd);
return;
}
/* send the nspace back to the tool */
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, cd->proc.nspace, PMIX_MAX_NSLEN+1))) {
PMIX_ERROR_LOG(rc);
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
PMIX_RELEASE(cd);
return;
}
/* send my nspace back to the tool */
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1))) {
PMIX_ERROR_LOG(rc);
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
PMIX_RELEASE(cd);
return;
}
/* send my rank back to the tool */
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&pmix_globals.myid.rank, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
PMIX_RELEASE(cd);
return;
}
/* add this nspace to our pool */
nptr = PMIX_NEW(pmix_nspace_t);
(void)strncpy(nptr->nspace, cd->proc.nspace, PMIX_MAX_NSLEN);
nptr->server = PMIX_NEW(pmix_server_nspace_t);
pmix_list_append(&pmix_globals.nspaces, &nptr->super);
/* add this tool rank to the nspace */
info = PMIX_NEW(pmix_rank_info_t);
PMIX_RETAIN(nptr);
info->nptr = nptr;
info->rank = 0;
/* need to include the uid/gid for validation */
info->uid = pnd->uid;
info->gid = pnd->gid;
pmix_list_append(&nptr->server->ranks, &info->super);
/* setup a peer object for this tool */
pmix_peer_t *peer = PMIX_NEW(pmix_peer_t);
PMIX_RETAIN(info);
peer->info = info;
peer->proc_cnt = 1;
peer->sd = pnd -> sd;
/* get the appropriate compatibility modules */
// peer->comm.type = pnd->buffer_type;
// peer->comm.bfrops = pmix_bfrops_base_assign_module(pnd->bfrop);
peer->compat.psec = pmix_psec_base_assign_module(pnd->psec);
/* if a credential is available, then check it */
if (NULL != peer->compat.psec->validate_cred) {
if (PMIX_SUCCESS != (rc = peer->compat.psec->validate_cred(peer, pnd->cred))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"validation of tool credential failed: %s",
PMIx_Error_string(rc));
PMIX_RELEASE(peer);
pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
PMIX_RELEASE(nptr); // will release the info object
CLOSE_THE_SOCKET(pnd->sd);
goto done;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"tool credential validated");
/* send them success */
rc = PMIX_SUCCESS;
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(peer);
pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
PMIX_RELEASE(nptr); // will release the info object
CLOSE_THE_SOCKET(pnd->sd);
goto done;
}
} else if (NULL != peer->compat.psec->server_handshake) {
/* execute the handshake if the security mode calls for it */
pmix_output_verbose(2, pmix_globals.debug_output,
"connect-ack executing handshake");
rc = PMIX_ERR_READY_FOR_HANDSHAKE;
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(sd, (char*)&rc, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(peer);
pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
PMIX_RELEASE(nptr); // will release the info object
CLOSE_THE_SOCKET(pnd->sd);
goto done;
}
if (PMIX_SUCCESS != (rc = peer->compat.psec->server_handshake(peer))) {
PMIX_ERROR_LOG(rc);
PMIX_RELEASE(peer);
pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
PMIX_RELEASE(nptr); // will release the info object
CLOSE_THE_SOCKET(pnd->sd);
goto done;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"tool connect-ack handshake complete");
} else {
/* this is not allowed */
PMIX_RELEASE(peer);
pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
PMIX_RELEASE(nptr); // will release the info object
CLOSE_THE_SOCKET(pnd->sd);
goto done;
}
/* set the socket non-blocking for all further operations */
pmix_usock_set_nonblocking(pnd->sd);
if (0 > (peer->index = pmix_pointer_array_add(&pmix_server_globals.clients, peer))) {
PMIX_RELEASE(pnd);
PMIX_RELEASE(cd);
PMIX_RELEASE(peer);
pmix_list_remove_item(&pmix_globals.nspaces, &nptr->super);
PMIX_RELEASE(nptr); // will release the info object
/* probably cannot send an error reply if we are out of memory */
return;
}
/* start the events for this tool */
event_assign(&peer->recv_event, pmix_globals.evbase, pnd->sd,
EV_READ|EV_PERSIST, pmix_usock_recv_handler, peer);
event_add(&peer->recv_event, NULL);
peer->recv_ev_active = true;
event_assign(&peer->send_event, pmix_globals.evbase, pnd->sd,
EV_WRITE|EV_PERSIST, pmix_usock_send_handler, peer);
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:server tool %s:%d has connected on socket %d",
peer->info->nptr->nspace, peer->info->rank, peer->sd);
done:
PMIX_RELEASE(pnd);
PMIX_RELEASE(cd);
}
/* receive a callback from the host RM with an nspace
* for a connecting tool */
static void cnct_cbfunc(pmix_status_t status,
pmix_proc_t *proc, void *cbdata)
{
pmix_setup_caddy_t *cd;
/* need to thread-shift this into our context */
cd = PMIX_NEW(pmix_setup_caddy_t);
cd->status = status;
(void)strncpy(cd->proc.nspace, proc->nspace, PMIX_MAX_NSLEN);
cd->cbdata = cbdata;
PMIX_THREADSHIFT(cd, process_cbfunc);
}
/* Parse init-ack message:
* NSPACE<0><rank>VERSION<0>BFROP<0>SEC<0>BUFTYPE<0>[CRED<0>]
*/
static pmix_status_t parse_connect_ack (char *msg,
pmix_listener_protocol_t protocol,
int len,
char **nspace, pmix_rank_t *rank,
char **version,
char **bfrop, char **sec,
pmix_bfrop_buffer_type_t *type,
char **cred)
{
int msglen;
if (PMIX_PROTOCOL_TOOL != protocol) {
PMIX_STRNLEN(msglen, msg, len);
if (msglen < len) {
*nspace = msg;
msg += strlen(*nspace) + 1;
len -= strlen(*nspace) + 1;
} else {
PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
return PMIX_ERR_BAD_PARAM;
}
if ((int)sizeof(pmix_rank_t) <= len) {
memcpy(rank, msg, sizeof(pmix_rank_t));
msg += sizeof(pmix_rank_t);
len -= sizeof(pmix_rank_t);
} else {
PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
return PMIX_ERR_BAD_PARAM;
}
}
PMIX_STRNLEN(msglen, msg, len);
if (msglen < len) {
*version = msg;
msg += strlen(*version) + 1;
len -= strlen(*version) + 1;
} else {
PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
return PMIX_ERR_BAD_PARAM;
}
#if 0
PMIX_STRNLEN(msglen, msg, len);
if (msglen < len) {
*bfrop = msg;
msg += strlen(*bfrop) + 1;
len -= strlen(*bfrop) + 1;
} else {
PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
return PMIX_ERR_BAD_PARAM;
}
#endif
PMIX_STRNLEN(msglen, msg, len);
if (msglen < len) {
*sec = msg;
msg += strlen(*sec) + 1;
len -= strlen(*sec) + 1;
} else {
PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
return PMIX_ERR_BAD_PARAM;
}
#if 0
if ((int)sizeof(pmix_bfrop_buffer_type_t) <= len) {
memcpy(type, msg, sizeof(pmix_bfrop_buffer_type_t));
msg += sizeof(pmix_bfrop_buffer_type_t);
len -= sizeof(pmix_bfrop_buffer_type_t);
} else {
PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM);
return PMIX_ERR_BAD_PARAM;
}
#endif
PMIX_STRNLEN(msglen, msg, len);
if (msglen < len)
*cred = msg;
else {
*cred = NULL;
}
return PMIX_SUCCESS;
}
/* Receive the peer's identification info from a newly
* connected socket and verify the expected response.
*/
static pmix_status_t pmix_server_authenticate(pmix_pending_connection_t *pnd,
pmix_rank_t *out_rank,
pmix_peer_t **peer)
{
char *msg, *nspace=NULL, *version=NULL, *bfrop=NULL, *sec=NULL, *cred=NULL;
pmix_status_t rc;
pmix_rank_t rank=PMIX_RANK_UNDEF;
pmix_usock_hdr_t hdr;
pmix_nspace_t *nptr, *tmp;
pmix_rank_info_t *info;
pmix_peer_t *psave = NULL;
bool found;
pmix_proc_t proc;
uid_t uid;
gid_t gid;
pmix_bfrop_buffer_type_t buffer_type = 0;
pmix_output_verbose(2, pmix_globals.debug_output,
"RECV CONNECT ACK FROM PEER ON SOCKET %d",
pnd->sd);
/* ensure all is zero'd */
memset(&hdr, 0, sizeof(pmix_usock_hdr_t));
if (NULL != peer) {
*peer = NULL;
}
/* get the header */
if (PMIX_SUCCESS != (rc = pmix_usock_recv_blocking(pnd->sd, (char*)&hdr, sizeof(pmix_usock_hdr_t)))) {
return rc;
}
/* get the id, authentication and version payload (and possibly
* security credential) - to guard against potential attacks,
* we'll set an arbitrary limit per a define */
if (PMIX_MAX_CRED_SIZE < hdr.nbytes) {
return PMIX_ERR_BAD_PARAM;
}
if (NULL == (msg = (char*)malloc(hdr.nbytes))) {
return PMIX_ERR_OUT_OF_RESOURCE;
}
if (PMIX_SUCCESS != pmix_usock_recv_blocking(pnd->sd, msg, hdr.nbytes)) {
/* unable to complete the recv */
pmix_output_verbose(2, pmix_globals.debug_output,
"unable to complete recv of connect-ack with client ON SOCKET %d",
pnd->sd);
free(msg);
return PMIX_ERR_UNREACH;
}
if (PMIX_SUCCESS != (rc = parse_connect_ack(msg, pnd->protocol,
hdr.nbytes, &nspace,
&rank, &version,
&bfrop, &sec,
&buffer_type, &cred))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"error parsing connect-ack from client ON SOCKET %d", pnd->sd);
free(msg);
return rc;
}
/* if the attaching process is a tool, then kickoff that procedure */
if (PMIX_PROTOCOL_TOOL == pnd->protocol) {
/* get the tool socket's uid and gid so we can pass them to
* the host RM for validation */
if (PMIX_SUCCESS != (rc = pmix_util_getid(pnd->sd, &uid, &gid))) {
free(msg);
return rc;
}
/* we pass this info in an array of pmix_info_t structs,
* so set that up here */
pnd->ninfo = 2;
PMIX_INFO_CREATE(pnd->info, pnd->ninfo);
(void)strncpy(pnd->info[0].key, PMIX_USERID, PMIX_MAX_KEYLEN);
pnd->info[0].value.type = PMIX_UINT32;
pnd->info[0].value.data.uint32 = uid;
(void)strncpy(pnd->info[1].key, PMIX_GRPID, PMIX_MAX_KEYLEN);
pnd->info[0].value.type = PMIX_UINT32;
pnd->info[0].value.data.uint32 = gid;
/* pass along the bfrop, buffer_type, and sec fields so
* we can assign them once we create a peer object */
pnd->uid = uid;
pnd->gid = gid;
if (NULL != bfrop) {
pnd->bfrop = strdup(bfrop);
}
if (NULL != sec) {
pnd->psec = strdup(sec);
}
pnd->buffer_type = buffer_type;
/* pass along the credential for later */
if (NULL != cred) {
pnd->cred = strdup(cred);
}
/* release the msg */
free(msg);
/* request an nspace for this requestor - it will
* automatically be assigned rank=0 */
pmix_host_server.tool_connected(pnd->info, pnd->ninfo, cnct_cbfunc, pnd);
return PMIX_ERR_OPERATION_IN_PROGRESS;
}
/* if the attaching process is not a tool, then set it up as
* a known peer */
pmix_output_verbose(2, pmix_globals.debug_output,
"connect-ack recvd from peer %s:%d:%s",
nspace, rank, version);
/* do not check the version - we only retain it at this
* time in case we need to check it at some future date.
* For now, our intent is to retain backward compatibility
* and so we will assume that all versions are compatible. */
/* see if we know this nspace */
nptr = NULL;
PMIX_LIST_FOREACH(tmp, &pmix_globals.nspaces, pmix_nspace_t) {
if (0 == strcmp(tmp->nspace, nspace)) {
nptr = tmp;
break;
}
}
if (NULL == nptr) {
/* we don't know this namespace, reject it */
free(msg);
/* send an error reply to the client */
rc = PMIX_ERR_NOT_FOUND;
goto error;
}
/* see if we have this peer in our list */
info = NULL;
found = false;
PMIX_LIST_FOREACH(info, &nptr->server->ranks, pmix_rank_info_t) {
if (info->rank == rank) {
found = true;
break;
}
}
if (!found) {
/* rank unknown, reject it */
free(msg);
/* send an error reply to the client */
rc = PMIX_ERR_NOT_FOUND;
goto error;
}
*out_rank = rank;
/* a peer can connect on multiple sockets since it can fork/exec
* a child that also calls PMIx_Init, so add it here if necessary.
* Create the tracker for this peer */
psave = PMIX_NEW(pmix_peer_t);
if (NULL == psave) {
rc = PMIX_ERR_NOMEM;
goto error;
}
PMIX_RETAIN(info);
psave->info = info;
info->proc_cnt++; /* increase number of processes on this rank */
psave->sd = pnd->sd;
if (0 > (psave->index = pmix_pointer_array_add(&pmix_server_globals.clients, psave))) {
free(msg);
PMIX_RELEASE(psave);
/* probably cannot send an error reply if we are out of memory */
return PMIX_ERR_OUT_OF_RESOURCE;
}
if (PMIX_PROTOCOL_V1 == pnd->protocol) {
/* we ignore the version string from 1.1 and before */
/* assign the 1.1 bfrops module to this peer */
// psave->compat.type = buffer_type;
// psave->compat.bfrops = pmix_bfrops_base_assign_module("1.1");
/* take the default sec module as that is what we
* told them to use */
psave->compat.psec = pmix_psec_base_assign_module(NULL);
} else if (PMIX_PROTOCOL_V2 == pnd->protocol) {
/* the newer protocol contains two fields:
* (a) a string containing the supported bfrops modules
* (b) a string containing the supported sec modules
*/
/* set the bfrops module to match this peer */
// psave->compat.type = buffer_type;
// psave->compat.bfrops = pmix_bfrops_base_assign_module(bfrop);
/* set the sec module to match this peer */
psave->compat.psec = pmix_psec_base_assign_module(sec);
} else {
/* unrecognized */
free(msg);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
PMIX_RELEASE(info);
return PMIX_ERR_BAD_PARAM;
}
free(msg);
/* see if there is a credential */
if (NULL != psave->compat.psec->validate_cred) {
if (PMIX_SUCCESS != (rc = psave->compat.psec->validate_cred(psave, cred))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"validation of client credential failed");
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
/* send an error reply to the client */
goto error;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"client credential validated");
/* send them success */
rc = PMIX_SUCCESS;
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
return rc;
}
} else if (NULL != pmix_globals.mypeer->compat.psec->server_handshake) {
pmix_output_verbose(2, pmix_globals.debug_output,
"connect-ack executing handshake");
rc = PMIX_ERR_READY_FOR_HANDSHAKE;
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
return rc;
}
if (PMIX_SUCCESS != (rc = pmix_globals.mypeer->compat.psec->server_handshake(psave))) {
PMIX_ERROR_LOG(rc);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
return rc;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"connect-ack handshake complete");
} else {
/* this is not allowed */
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
/* send an error reply to the client */
rc = PMIX_ERR_INVALID_CRED;
goto error;
}
/* send the client's array index */
if (PMIX_SUCCESS != (rc = pmix_usock_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) {
PMIX_ERROR_LOG(rc);
pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL);
PMIX_RELEASE(psave);
return rc;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"connect-ack from client completed");
*peer = psave;
/* let the host server know that this client has connected */
if (NULL != pmix_host_server.client_connected) {
(void)strncpy(proc.nspace, psave->info->nptr->nspace, PMIX_MAX_NSLEN);
proc.rank = psave->info->rank;
rc = pmix_host_server.client_connected(&proc, psave->info->server_object,
NULL, NULL);
if (PMIX_SUCCESS != rc) {
PMIX_ERROR_LOG(rc);
}
}
return rc;
error:
/* send an error reply to the client */
if (PMIX_SUCCESS != pmix_usock_send_blocking(pnd->sd, (char*)&rc, sizeof(int))) {
PMIX_ERROR_LOG(rc);
}
return rc;
}
/*
* Handler for accepting client connections from the event library
*/
static void connection_handler(int sd, short flags, void* cbdata)
{
pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata;
pmix_peer_t *peer;
pmix_rank_t rank;
pmix_status_t status;
pmix_output_verbose(8, pmix_globals.debug_output,
"connection_handler: new connection: %d",
pnd->sd);
/* ensure the socket is in blocking mode */
pmix_usock_set_blocking(pnd->sd);
/*
* Receive identifier info from the client and authenticate it - the
* function will lookup and return the peer object if the connection
* is successfully authenticated */
if (PMIX_SUCCESS != (status = pmix_server_authenticate(pnd, &rank, &peer))) {
if (PMIX_ERR_OPERATION_IN_PROGRESS != status) {
CLOSE_THE_SOCKET(pnd->sd);
}
return;
}
pmix_usock_set_nonblocking(pnd->sd);
/* start the events for this client */
event_assign(&peer->recv_event, pmix_globals.evbase, pnd->sd,
EV_READ|EV_PERSIST, pmix_usock_recv_handler, peer);
event_add(&peer->recv_event, NULL);
peer->recv_ev_active = true;
event_assign(&peer->send_event, pmix_globals.evbase, pnd->sd,
EV_WRITE|EV_PERSIST, pmix_usock_send_handler, peer);
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix:server client %s:%u has connected on socket %d",
peer->info->nptr->nspace, peer->info->rank, peer->sd);
PMIX_RELEASE(pnd);
}
/*
* Handler for accepting tool connections from the event library
*/
static void tool_handler(int sd, short flags, void* cbdata)
{
pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata;
pmix_output_verbose(1, pmix_globals.debug_output,
"tool_handler: new tool connection: %d",
pnd->sd);
/* if the server doesn't support this, then abort now */
if (NULL == pmix_host_server.tool_connected) {
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
}
/* ensure the socket is in blocking mode */
pmix_usock_set_blocking(pnd->sd);
/* initiate the authentication handshake */
if (PMIX_ERR_OPERATION_IN_PROGRESS != pmix_server_authenticate(pnd, NULL, NULL)) {
CLOSE_THE_SOCKET(pnd->sd);
PMIX_RELEASE(pnd);
}
}

64
opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.c
Просмотреть файл

@ -54,7 +54,6 @@
#include "src/util/error.h"
#include "src/util/output.h"
#include "src/util/pmix_environ.h"
#include "src/usock/usock.h"
#include "pmix_server_ops.h"
@ -1594,34 +1593,6 @@ PMIX_CLASS_INSTANCE(pmix_dmdx_local_t,
pmix_list_item_t,
lmcon, lmdes);
static void pccon(pmix_pending_connection_t *p)
{
memset(p->nspace, 0, PMIX_MAX_NSLEN+1);
p->info = NULL;
p->ninfo = 0;
p->bfrop = NULL;
p->psec = NULL;
p->cred = NULL;
}
static void pcdes(pmix_pending_connection_t *p)
{
if (NULL != p->info) {
PMIX_INFO_FREE(p->info, p->ninfo);
}
if (NULL != p->bfrop) {
free(p->bfrop);
}
if (NULL != p->psec) {
free(p->psec);
}
if (NULL != p->cred) {
free(p->cred);
}
}
PMIX_CLASS_INSTANCE(pmix_pending_connection_t,
pmix_object_t,
pccon, pcdes);
static void prevcon(pmix_peer_events_info_t *p)
{
p->peer = NULL;
@ -1647,38 +1618,3 @@ static void regdes(pmix_regevents_info_t *p)
PMIX_CLASS_INSTANCE(pmix_regevents_info_t,
pmix_list_item_t,
regcon, regdes);
static void lcon(pmix_listener_t *p)
{
memset(&p->address, 0, sizeof(struct sockaddr_un));
p->address.sun_family = AF_UNIX;
p->socket = -1;
p->varname = NULL;
p->uri = NULL;
p->owner_given = false;
p->group_given = false;
p->mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH;
}
static void ldes(pmix_listener_t *p)
{
if (0 <= p->socket) {
CLOSE_THE_SOCKET(p->socket);
}
/* cleanup the rendezvous file */
if (0 == access(p->address.sun_path, F_OK)) {
unlink(p->address.sun_path);
}
if (NULL != p->varname) {
free(p->varname);
}
if (NULL != p->uri) {
free(p->uri);
}
}
PMIX_CLASS_INSTANCE(pmix_listener_t,
pmix_list_item_t,
lcon, ldes);
PMIX_CLASS_INSTANCE(pmix_usock_queue_t,
pmix_object_t,
NULL, NULL);

92
opal/mca/pmix/pmix2x/pmix/src/server/pmix_server_ops.h
Просмотреть файл

@ -19,7 +19,6 @@
#include <pmix_common.h>
#include <src/class/pmix_ring_buffer.h>
#include <pmix_server.h>
#include "src/usock/usock.h"
#include "src/util/hash.h"
typedef struct {
@ -86,32 +85,6 @@ typedef struct {
} pmix_dmdx_local_t;
PMIX_CLASS_DECLARATION(pmix_dmdx_local_t);
/* define listener protocol types */
typedef uint16_t pmix_listener_protocol_t;
#define PMIX_PROTOCOL_V1 0
#define PMIX_PROTOCOL_TOOL 1
#define PMIX_PROTOCOL_V2 2
/* connection support */
typedef struct {
pmix_object_t super;
pmix_event_t ev;
pmix_listener_protocol_t protocol;
int sd;
char nspace[PMIX_MAX_NSLEN+1];
pmix_info_t *info;
size_t ninfo;
pmix_status_t status;
struct sockaddr_storage addr;
char *bfrop;
char *psec;
pmix_bfrop_buffer_type_t buffer_type;
char *cred;
uid_t uid;
gid_t gid;
} pmix_pending_connection_t;
PMIX_CLASS_DECLARATION(pmix_pending_connection_t);
/* event/error registration book keeping */
typedef struct {
pmix_list_item_t super;
@ -127,45 +100,17 @@ typedef struct {
} pmix_regevents_info_t;
PMIX_CLASS_DECLARATION(pmix_regevents_info_t);
/* listener objects */
typedef struct pmix_listener_t {
pmix_list_item_t super;
pmix_listener_protocol_t protocol;
int socket;
struct sockaddr_un address;
char *varname;
char *uri;
uint32_t owner;
bool owner_given;
uint32_t group;
bool group_given;
uint32_t mode;
} pmix_listener_t;
PMIX_CLASS_DECLARATION(pmix_listener_t);
typedef struct {
pmix_pointer_array_t clients; // array of pmix_peer_t local clients
pmix_list_t collectives; // list of active pmix_server_trkr_t
pmix_list_t remote_pnd; // list of pmix_dmdx_remote_t awaiting arrival of data fror servicing remote req's
pmix_list_t local_reqs; // list of pmix_dmdx_local_t awaiting arrival of data from local neighbours
volatile bool listen_thread_active; // listen thread is running
pmix_list_t listeners; // list of pmix_listener_t
int stop_thread[2]; // pipe used to stop listener thread
pmix_buffer_t gdata; // cache of data given to me for passing to all clients
pmix_list_t events; // list of pmix_regevents_info_t registered events
pmix_ring_buffer_t notifications; // ring buffer of pending notifications
bool tool_connections_allowed;
} pmix_server_globals_t;
typedef struct {
pmix_object_t super;
pmix_event_t ev;
pmix_peer_t *peer;
pmix_buffer_t *buf;
uint32_t tag;
} pmix_usock_queue_t;
PMIX_CLASS_DECLARATION(pmix_usock_queue_t);
#define PMIX_PEER_CADDY(c, p, t) \
do { \
(c) = PMIX_NEW(pmix_server_caddy_t); \
@ -174,14 +119,6 @@ PMIX_CLASS_DECLARATION(pmix_usock_queue_t);
(c)->peer = (p); \
} while (0)
#define PMIX_SND_CADDY(c, h, s) \
do { \
(c) = PMIX_NEW(pmix_server_caddy_t); \
(void)memcpy(&(c)->hdr, &(h), sizeof(pmix_usock_hdr_t)); \
PMIX_RETAIN((s)); \
(c)->snd = (s); \
} while (0)
#define PMIX_SETUP_COLLECTIVE(c, t) \
do { \
(c) = PMIX_NEW(pmix_trkr_caddy_t); \
@ -197,33 +134,6 @@ PMIX_CLASS_DECLARATION(pmix_usock_queue_t);
} while (0)
/* queue a message to be sent to one of our procs - must
* provide the following params:
*
* t - tag to be sent to
* b - buffer to be sent
*/
#define PMIX_SERVER_QUEUE_REPLY(p, t, b) \
do { \
pmix_usock_queue_t *queue; \
queue = PMIX_NEW(pmix_usock_queue_t); \
queue->peer = (p); \
queue->buf = (b); \
queue->tag = (t); \
pmix_output_verbose(2, pmix_globals.debug_output, \
"[%s:%d] queue reply to %s:%d on tag %d", \
__FILE__, __LINE__, \
(queue->peer)->info->nptr->nspace, \
(queue->peer)->info->rank, (queue->tag)); \
event_assign(&queue->ev, pmix_globals.evbase, -1, \
EV_WRITE, pmix_server_queue_message, queue); \
event_priority_set(&queue->ev, 0); \
event_active(&queue->ev, EV_WRITE, 1); \
} while (0)
pmix_status_t pmix_prepare_listening(pmix_listener_t *lt, bool *need_listener);
pmix_status_t pmix_start_listening(void);
void pmix_stop_listening(void);
bool pmix_server_trk_update(pmix_server_trkr_t *trk);
@ -305,8 +215,6 @@ pmix_status_t pmix_server_event_recvd_from_client(pmix_peer_t *peer,
void *cbdata);
void pmix_server_execute_collective(int sd, short args, void *cbdata);
void pmix_server_queue_message(int fd, short args, void *cbdata);
extern pmix_server_module_t pmix_host_server;
extern pmix_server_globals_t pmix_server_globals;

457
opal/mca/pmix/pmix2x/pmix/src/tool/pmix_tool.c
Просмотреть файл

@ -68,7 +68,7 @@ extern pmix_client_globals_t pmix_client_globals;
#include "src/util/output.h"
#include "src/runtime/pmix_progress_threads.h"
#include "src/runtime/pmix_rte.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/ptl.h"
#include "src/mca/psec/psec.h"
#include "src/include/pmix_globals.h"
#if defined(PMIX_ENABLE_DSTORE) && (PMIX_ENABLE_DSTORE == 1)
@ -77,18 +77,14 @@ extern pmix_client_globals_t pmix_client_globals;
#define PMIX_MAX_RETRIES 10
static char *mytmpdir = NULL;
static char *systmpdir = NULL;
static pmix_status_t usock_connect(struct sockaddr_un *address, int *fd);
static void _notify_complete(pmix_status_t status, void *cbdata)
{
pmix_event_chain_t *chain = (pmix_event_chain_t*)cbdata;
PMIX_RELEASE(chain);
}
static void pmix_tool_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *hdr,
static void pmix_tool_notify_recv(struct pmix_peer_t *peer,
pmix_ptl_hdr_t *hdr,
pmix_buffer_t *buf, void *cbdata)
{
pmix_status_t rc;
@ -154,55 +150,13 @@ static void pmix_tool_notify_recv(struct pmix_peer_t *peer, pmix_usock_hdr_t *hd
}
static pmix_status_t connect_to_server(struct sockaddr_un *address)
{
int sd;
pmix_status_t ret;
if (PMIX_SUCCESS != (ret = usock_connect(address, &sd))) {
PMIX_ERROR_LOG(ret);
return ret;
}
pmix_client_globals.myserver.sd = sd;
/* setup recv event */
event_assign(&pmix_client_globals.myserver.recv_event,
pmix_globals.evbase,
pmix_client_globals.myserver.sd,
EV_READ | EV_PERSIST,
pmix_usock_recv_handler, &pmix_client_globals.myserver);
event_add(&pmix_client_globals.myserver.recv_event, 0);
pmix_client_globals.myserver.recv_ev_active = true;
/* setup send event */
event_assign(&pmix_client_globals.myserver.send_event,
pmix_globals.evbase,
pmix_client_globals.myserver.sd,
EV_WRITE|EV_PERSIST,
pmix_usock_send_handler, &pmix_client_globals.myserver);
pmix_client_globals.myserver.send_ev_active = false;
return PMIX_SUCCESS;
}
PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
pmix_info_t info[], size_t ninfo)
{
char *evar, *tdir, *tmp;
struct sockaddr_un address;
size_t n;
pmix_kval_t *kptr;
pmix_status_t rc;
pmix_nspace_t *nptr, *nsptr;
pid_t server_pid=0;
bool server_pid_given = false;
int hostnamelen = 30;
char hostname[hostnamelen];
DIR *cur_dirp = NULL;
struct dirent * dir_entry;
bool connect_to_system_server = false;
bool connect_to_system_first = false;
bool connection_defined = false;
char hostname[PMIX_MAX_NSLEN];
if (NULL == proc) {
return PMIX_ERR_BAD_PARAM;
@ -220,28 +174,6 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
return PMIX_SUCCESS;
}
/* scan incoming info for directives */
if (NULL != info) {
for (n=0; n < ninfo; n++) {
if (strcmp(info[n].key, PMIX_SERVER_PIDINFO) == 0) {
server_pid = info[n].value.data.pid;
server_pid_given = true;
} else if (strcmp(info[n].key, PMIX_CONNECT_TO_SYSTEM) == 0) {
connect_to_system_server = info[n].value.data.flag;
connection_defined = true;
} else if (strcmp(info[n].key, PMIX_CONNECT_SYSTEM_FIRST) == 0) {
connect_to_system_first = info[n].value.data.flag;
connection_defined = true;
} else if (strcmp(info[n].key, PMIX_SERVER_TMPDIR) == 0 &&
NULL == mytmpdir) {
mytmpdir = strdup(info[n].value.data.string);
} else if (strcmp(info[n].key, PMIX_SYSTEM_TMPDIR) == 0 &&
NULL == systmpdir) {
systmpdir = strdup(info[n].value.data.string);
}
}
}
/* setup the runtime - this init's the globals,
* opens and initializes the required frameworks */
if (PMIX_SUCCESS != (rc = pmix_rte_init(PMIX_PROC_TOOL, info, ninfo,
@ -256,112 +188,20 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: init called");
/* setup the path to the daemon rendezvous point */
memset(&address, 0, sizeof(struct sockaddr_un));
address.sun_family = AF_UNIX;
/* Get hostname to match what the server is doing */
gethostname(hostname, hostnamelen);
/* ensure it is NULL terminated */
hostname[hostnamelen-1] = '\0';
/* if we are to connect solely to the system-level daemon,
* or to preferentially connect to the system-level daemon,
* or nothing was specified at all, then look to see if a
* rendezvous point in that location exists */
if (connect_to_system_server || connect_to_system_first || !connection_defined) {
/* find the temp dir */
if (NULL != systmpdir) {
tdir = systmpdir;
} else if (NULL == (tdir = getenv("TMPDIR"))) {
if (NULL == (tdir = getenv("TEMP"))) {
if (NULL == (tdir = getenv("TMP"))) {
tdir = "/tmp";
}
}
}
snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/pmix.sys.%s", tdir, hostname);
/* see if the rendezvous file exists */
if (0 != access(address.sun_path, R_OK)) {
/* if it was a requirement, then error out */
if (connect_to_system_server) {
return PMIX_ERR_UNREACH;
}
/* otherwise, this isn't a fatal error - reset the addr */
memset(&address, 0, sizeof(struct sockaddr_un));
address.sun_family = AF_UNIX;
connection_defined = false;
} else {
/* connect to this server */
connection_defined = true;
}
/* select our psec module - we take the default as we cannot
* do any better */
if (PMIX_SUCCESS != (rc = pmix_psec.assign_module(pmix_globals.mypeer, NULL))) {
return PMIX_ERR_INIT;
}
/* the server will have to use the same */
pmix_client_globals.myserver.compat.psec = pmix_globals.mypeer->compat.psec;
if (!connection_defined) {
/* if we get here, then either we are to connect to
* a non-system daemon, or a system-level daemon was
* not found - so now look for the session daemon */
/* find the temp dir */
if (NULL != mytmpdir) {
tdir = mytmpdir;
} else if (NULL == (tdir = getenv("TMPDIR"))) {
if (NULL == (tdir = getenv("TEMP"))) {
if (NULL == (tdir = getenv("TMP"))) {
tdir = "/tmp";
}
}
}
/* if they gave us a specific pid, then look for that
* particular server - otherwise, see if there is only
* one on this node and default to it */
if (server_pid_given) {
snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/pmix.%s.%d", tdir, hostname, server_pid);
/* if the rendezvous file doesn't exist, that's an error */
if (0 != access(address.sun_path, R_OK)) {
return PMIX_ERR_NOT_FOUND;
}
} else {
/* open up the temp directory */
if (NULL == (cur_dirp = opendir(tdir))) {
return PMIX_ERR_NOT_FOUND;
}
/* search the entries for something that starts with pmix.hostname */
if (0 > asprintf(&tmp, "pmix.%s", hostname)) {
closedir(cur_dirp);
return PMIX_ERR_NOMEM;
}
evar = NULL;
while (NULL != (dir_entry = readdir(cur_dirp))) {
if (0 == strncmp(dir_entry->d_name, tmp, strlen(tmp))) {
/* found one - if more than one, then that's an error */
if (NULL != evar) {
closedir(cur_dirp);
free(evar);
free(tmp);
return PMIX_ERR_INIT;
}
evar = strdup(dir_entry->d_name);
}
}
free(tmp);
closedir(cur_dirp);
if (NULL == evar) {
/* none found */
return PMIX_ERR_INIT;
}
/* use the found one as our contact point */
snprintf(address.sun_path, sizeof(address.sun_path)-1, "%s/%s", tdir, evar);
free(evar);
}
}
/* connect to the server */
if (PMIX_SUCCESS != (rc = connect_to_server(&address))) {
PMIX_ERROR_LOG(rc);
/* connect to the server - returns job info if successful */
if (PMIX_SUCCESS != (rc = pmix_ptl.connect_to_peer(&pmix_client_globals.myserver, info, ninfo))){
return rc;
}
pmix_output(0, "TOOL: SERVER CONNECTION COMPLETE");
/* increment our init reference counter */
pmix_globals.init_cntr++;
@ -381,6 +221,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc,
}
}
if (NULL == nsptr) {
pmix_output(0, "TOOL: NSPACE NOT FOUND");
return PMIX_ERR_NOT_FOUND;
}
@ -632,275 +473,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void)
PMIX_DESTRUCT(&pmix_client_globals.myserver);
PMIX_LIST_DESTRUCT(&pmix_client_globals.pending_requests);
if (NULL != mytmpdir) {
free(mytmpdir);
}
if (NULL != systmpdir) {
free(systmpdir);
}
pmix_globals_finalize();
pmix_class_finalize();
return PMIX_SUCCESS;
}
/*
* The sections below need to be updated to reflect tool
* connection handshake protocols - in this case, we
* don't know our nspace/rank in advance. So we need
* the handshake to include the security credential
* exchange, and then get our nspace/rank in return */
static pmix_status_t send_connect_ack(int sd)
{
char *msg;
pmix_usock_hdr_t hdr;
size_t sdsize=0, csize=0;
char *cred = NULL;
char *sec;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: TOOL SEND CONNECT ACK");
/* setup the header */
memset(&hdr, 0, sizeof(pmix_usock_hdr_t));
hdr.pindex = -1;
hdr.tag = UINT32_MAX;
/* get a credential, if the security system provides one. Not
* every SPC will do so, thus we must first check */
if (NULL != pmix_globals.mypeer->compat.psec->create_cred) {
if (NULL == (cred = pmix_globals.mypeer->compat.psec->create_cred())) {
/* an error occurred - we cannot continue */
return PMIX_ERR_INVALID_CRED;
}
csize = strlen(cred) + 1; // must NULL terminate the string!
}
/* add our active bfrops and sec module info, and what type
* of buffers we are using */
// bfrop = pmix_globals.mypeer->compat.bfrops->name;
sec = pmix_globals.mypeer->compat.psec->name;
/* set the number of bytes to be read beyond the header */
hdr.nbytes = strlen(PMIX_VERSION) + 1 + strlen(sec) + 1 + csize; // must NULL terminate the VERSION string!
/* create a space for our message */
sdsize = (sizeof(hdr) + hdr.nbytes);
if (NULL == (msg = (char*)malloc(sdsize))) {
if (NULL != cred) {
free(cred);
}
return PMIX_ERR_OUT_OF_RESOURCE;
}
memset(msg, 0, sdsize);
csize=0;
memcpy(msg, &hdr, sizeof(pmix_usock_hdr_t));
csize += sizeof(pmix_usock_hdr_t);
/* load the message */
memcpy(msg+csize, PMIX_VERSION, strlen(PMIX_VERSION));
csize += strlen(PMIX_VERSION)+1;
// memcpy(msg+csize, bfrop, strlen(bfrop));
// csize += strlen(bfrop)+1;
memcpy(msg+csize, sec, strlen(sec));
csize += strlen(sec)+1;
// memcpy(msg+csize, &pmix_globals.mypeer->compat.type, sizeof(pmix_bfrop_buffer_type_t));
// csize += sizeof(pmix_bfrop_buffer_type_t);
if (NULL != cred) {
memcpy(msg+csize, cred, strlen(cred)); // leaves last position in msg set to NULL
}
if (PMIX_SUCCESS != pmix_usock_send_blocking(sd, msg, sdsize)) {
free(msg);
if (NULL != cred) {
free(cred);
}
return PMIX_ERR_UNREACH;
}
free(msg);
if (NULL != cred) {
free(cred);
}
return PMIX_SUCCESS;
}
/* we receive a connection acknowledgement from the server,
* consisting of the status and (if success) the nspace assigned
* to us */
static pmix_status_t recv_connect_ack(int sd)
{
pmix_status_t reply;
struct timeval tv, save;
pmix_socklen_t sz;
bool sockopt = true;
pmix_nspace_t *nsptr;
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT ACK FROM SERVER");
/* get the current timeout value so we can reset to it */
sz = sizeof(save);
if (0 != getsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, (void*)&save, &sz)) {
if (ENOPROTOOPT == errno) {
sockopt = false;
} else {
return PMIX_ERR_UNREACH;
}
} else {
/* set a timeout on the blocking recv so we don't hang */
tv.tv_sec = 2;
tv.tv_usec = 0;
if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))) {
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: recv_connect_ack could not setsockopt SO_RCVTIMEO");
return PMIX_ERR_UNREACH;
}
}
/* get the returned status from the request for namespace */
pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t));
if (PMIX_SUCCESS != reply) {
return reply;
}
/* get our assigned nspace */
pmix_usock_recv_blocking(sd, pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1);
/* setup required bookkeeping */
nsptr = PMIX_NEW(pmix_nspace_t);
(void)strncpy(nsptr->nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN);
pmix_list_append(&pmix_globals.nspaces, &nsptr->super);
/* our rank is always zero */
pmix_globals.myid.rank = 0;
/* get the server's nspace and rank so we can send to it */
pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t);
pmix_client_globals.myserver.info->nptr = PMIX_NEW(pmix_nspace_t);
pmix_usock_recv_blocking(sd, (char*)pmix_client_globals.myserver.info->nptr->nspace, PMIX_MAX_NSLEN+1);
pmix_usock_recv_blocking(sd, (char*)&(pmix_client_globals.myserver.info->rank), sizeof(int));
pmix_output_verbose(2, pmix_globals.debug_output,
"pmix: RECV CONNECT CONFIRMATION FOR TOOL %s:%d FROM SERVER %s:%d",
pmix_globals.myid.nspace, pmix_globals.myid.rank,
pmix_client_globals.myserver.info->nptr->nspace,
pmix_client_globals.myserver.info->rank);
/* get the returned status from the security handshake */
pmix_usock_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t));
if (PMIX_SUCCESS != reply) {
/* see if they want us to do the handshake */
if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) {
if (NULL == pmix_globals.mypeer->compat.psec->client_handshake) {
return PMIX_ERR_HANDSHAKE_FAILED;
}
if (PMIX_SUCCESS != (reply = pmix_globals.mypeer->compat.psec->client_handshake(sd))) {
return reply;
}
/* if the handshake succeeded, then fall thru to the next step */
} else {
return reply;
}
}
if (sockopt) {
if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) {
return PMIX_ERR_UNREACH;
}
}
return PMIX_SUCCESS;
}
static pmix_status_t usock_connect(struct sockaddr_un *addr, int *fd)
{
int sd=-1;
pmix_status_t rc;
pmix_socklen_t addrlen = 0;
int retries = 0;
pmix_output_verbose(2, pmix_globals.debug_output,
"usock_peer_try_connect: attempting to connect to server at %s",
addr->sun_path);
addrlen = sizeof(struct sockaddr_un);
while (retries < PMIX_MAX_RETRIES) {
retries++;
/* Create the new socket */
sd = socket(PF_UNIX, SOCK_STREAM, 0);
if (sd < 0) {
pmix_output(0, "pmix:create_socket: socket() failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
continue;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"usock_peer_try_connect: attempting to connect to server on socket %d", sd);
/* try to connect */
int err = -1;
if ((err = connect(sd, (struct sockaddr*)addr, addrlen)) < 0) {
if (pmix_socket_errno == ETIMEDOUT) {
/* The server may be too busy to accept new connections */
pmix_output_verbose(2, pmix_globals.debug_output,
"timeout connecting to server");
CLOSE_THE_SOCKET(sd);
continue;
} else if (ECONNABORTED == pmix_socket_errno) {
/* Some kernels (Linux 2.6) will automatically software
abort a connection that was ECONNREFUSED on the last
attempt, without even trying to establish the
connection. Handle that case in a semi-rational
way by trying twice before giving up */
pmix_output_verbose(2, pmix_globals.debug_output,
"connection to server aborted by OS - retrying");
CLOSE_THE_SOCKET(sd);
continue;
} else {
pmix_output_verbose(2, pmix_globals.debug_output,
"Failed to connect, errno = %d, err= %s\n", errno, strerror(errno));
CLOSE_THE_SOCKET(sd);
continue;
}
}
/* otherwise, the connect succeeded - so break out of the loop */
break;
}
if (retries == PMIX_MAX_RETRIES || sd < 0){
/* We were unsuccessful in establishing this connection, and are
* not likely to suddenly become successful */
if (0 <= sd) {
CLOSE_THE_SOCKET(sd);
}
return PMIX_ERR_UNREACH;
}
/* send any authentication credentials to the server */
if (PMIX_SUCCESS != (rc = send_connect_ack(sd))) {
PMIX_ERROR_LOG(rc);
CLOSE_THE_SOCKET(sd);
return rc;
}
/* do whatever handshake is required */
if (PMIX_SUCCESS != (rc = recv_connect_ack(sd))) {
PMIX_ERROR_LOG(rc);
CLOSE_THE_SOCKET(sd);
return rc;
}
pmix_output_verbose(2, pmix_globals.debug_output,
"sock_peer_try_connect: Connection across to server succeeded");
/* mark the connection as made */
pmix_globals.connected = true;
pmix_usock_set_nonblocking(sd);
*fd = sd;
return PMIX_SUCCESS;
}

17
opal/mca/pmix/pmix2x/pmix/src/usock/Makefile.am
Просмотреть файл

@ -1,17 +0,0 @@
#
# Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
# Copyright (c) 2014 Artem Y. Polyakov <artpol84@gmail.com>.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
headers += \
usock/usock.h
sources += \
usock/usock.c \
usock/usock_sendrecv.c

348
opal/mca/pmix/pmix2x/pmix/src/usock/usock.c
Просмотреть файл

@ -1,348 +0,0 @@
/*
* Copyright (c) 2014-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2014 Artem Y. Polyakov <artpol84@gmail.com>.
* All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2016 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <src/include/pmix_config.h>
#include <src/include/types.h>
#include <src/include/pmix_stdint.h>
#include <src/include/pmix_socket_errno.h>
#include "src/include/pmix_globals.h"
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#include <fcntl.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_UN_H
#include <sys/un.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include "src/buffer_ops/buffer_ops.h"
#include "src/util/output.h"
#include "usock.h"
/* instance usock globals */
pmix_usock_globals_t pmix_usock_globals = {{{0}}};
void pmix_usock_init(pmix_usock_cbfunc_t cbfunc)
{
pmix_usock_posted_recv_t *req;
/* setup the usock globals */
PMIX_CONSTRUCT(&pmix_usock_globals.posted_recvs, pmix_list_t);
/* if a cbfunc was given, post a persistent recv
* for the special 0 tag so the client can recv
* error notifications from the server */
if (NULL != cbfunc) {
req = PMIX_NEW(pmix_usock_posted_recv_t);
req->tag = 0;
req->cbfunc = cbfunc;
pmix_output_verbose(5, pmix_globals.debug_output,
"posting notification recv on tag %d", req->tag);
/* add it to the list of recvs - we cannot have unexpected messages
* in this subsystem as the server never sends us something that
* we didn't previously request */
pmix_list_prepend(&pmix_usock_globals.posted_recvs, &req->super);
}
}
void pmix_usock_finalize(void)
{
PMIX_LIST_DESTRUCT(&pmix_usock_globals.posted_recvs);
}
pmix_status_t pmix_usock_set_nonblocking(int sd)
{
int flags;
/* setup the socket as non-blocking */
if ((flags = fcntl(sd, F_GETFL, 0)) < 0) {
pmix_output(0, "usock_peer_connect: fcntl(F_GETFL) failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
} else {
flags |= O_NONBLOCK;
if(fcntl(sd, F_SETFL, flags) < 0)
pmix_output(0, "usock_peer_connect: fcntl(F_SETFL) failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
}
return PMIX_SUCCESS;
}
pmix_status_t pmix_usock_set_blocking(int sd)
{
int flags;
/* setup the socket as non-blocking */
if ((flags = fcntl(sd, F_GETFL, 0)) < 0) {
pmix_output(0, "usock_peer_connect: fcntl(F_GETFL) failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
} else {
flags &= ~(O_NONBLOCK);
if(fcntl(sd, F_SETFL, flags) < 0)
pmix_output(0, "usock_peer_connect: fcntl(F_SETFL) failed: %s (%d)\n",
strerror(pmix_socket_errno),
pmix_socket_errno);
}
return PMIX_SUCCESS;
}
/*
* A blocking send on a non-blocking socket. Used to send the small amount of connection
* information that identifies the peers endpoint.
*/
pmix_status_t pmix_usock_send_blocking(int sd, char *ptr, size_t size)
{
size_t cnt = 0;
int retval;
pmix_output_verbose(8, pmix_globals.debug_output,
"send blocking of %"PRIsize_t" bytes to socket %d",
size, sd );
while (cnt < size) {
retval = send(sd, (char*)ptr+cnt, size-cnt, 0);
if (retval < 0) {
if (EAGAIN == pmix_socket_errno ||
EWOULDBLOCK == pmix_socket_errno) {
/* just cycle and let it try again */
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking_send received error %d:%s from remote - cycling",
pmix_socket_errno, strerror(pmix_socket_errno));
continue;
}
if (pmix_socket_errno != EINTR) {
pmix_output_verbose(8, pmix_globals.debug_output,
"usock_peer_send_blocking: send() to socket %d failed: %s (%d)\n",
sd, strerror(pmix_socket_errno),
pmix_socket_errno);
return PMIX_ERR_UNREACH;
}
continue;
}
cnt += retval;
}
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking send complete to socket %d", sd);
return PMIX_SUCCESS;
}
/*
* A blocking recv on a non-blocking socket. Used to receive the small amount of connection
* information that identifies the peers endpoint.
*/
pmix_status_t pmix_usock_recv_blocking(int sd, char *data, size_t size)
{
size_t cnt = 0;
pmix_output_verbose(8, pmix_globals.debug_output,
"waiting for blocking recv of %"PRIsize_t" bytes", size);
while (cnt < size) {
int retval = recv(sd, (char *)data+cnt, size-cnt, MSG_WAITALL);
/* remote closed connection */
if (retval == 0) {
pmix_output_verbose(8, pmix_globals.debug_output,
"usock_recv_blocking: remote closed connection");
return PMIX_ERR_UNREACH;
}
/* handle errors */
if (retval < 0) {
if (EAGAIN == pmix_socket_errno ||
EWOULDBLOCK == pmix_socket_errno) {
/* just cycle and let it try again */
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking_recv received error %d:%s from remote - cycling",
pmix_socket_errno, strerror(pmix_socket_errno));
continue;
}
if (pmix_socket_errno != EINTR ) {
/* If we overflow the listen backlog, it's
possible that even though we finished the three
way handshake, the remote host was unable to
transition the connection from half connected
(received the initial SYN) to fully connected
(in the listen backlog). We likely won't see
the failure until we try to receive, due to
timing and the like. The first thing we'll get
in that case is a RST packet, which receive
will turn into a connection reset by peer
errno. In that case, leave the socket in
CONNECT_ACK and propogate the error up to
recv_connect_ack, who will try to establish the
connection again */
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking_recv received error %d:%s from remote - aborting",
pmix_socket_errno, strerror(pmix_socket_errno));
return PMIX_ERR_UNREACH;
}
continue;
}
cnt += retval;
}
pmix_output_verbose(8, pmix_globals.debug_output,
"blocking receive complete from remote");
return PMIX_SUCCESS;
}
/*** INSTANTIATE INTERNAL CLASSES ***/
static void scon(pmix_usock_send_t *p)
{
memset(&p->hdr, 0, sizeof(pmix_usock_hdr_t));
p->hdr.tag = UINT32_MAX;
p->hdr.nbytes = 0;
p->data = NULL;
p->hdr_sent = false;
p->sdptr = NULL;
p->sdbytes = 0;
}
static void sdes(pmix_usock_send_t *p)
{
if (NULL != p->data) {
PMIX_RELEASE(p->data);
}
}
PMIX_CLASS_INSTANCE(pmix_usock_send_t,
pmix_list_item_t,
scon, sdes);
static void rcon(pmix_usock_recv_t *p)
{
memset(&p->hdr, 0, sizeof(pmix_usock_hdr_t));
p->hdr.tag = UINT32_MAX;
p->hdr.nbytes = 0;
p->data = NULL;
p->hdr_recvd = false;
p->rdptr = NULL;
p->rdbytes = 0;
}
PMIX_CLASS_INSTANCE(pmix_usock_recv_t,
pmix_list_item_t,
rcon, NULL);
static void prcon(pmix_usock_posted_recv_t *p)
{
p->tag = UINT32_MAX;
p->cbfunc = NULL;
p->cbdata = NULL;
}
PMIX_CLASS_INSTANCE(pmix_usock_posted_recv_t,
pmix_list_item_t,
prcon, NULL);
static void cbcon(pmix_cb_t *p)
{
p->active = false;
p->checked = false;
PMIX_CONSTRUCT(&p->data, pmix_buffer_t);
p->cbfunc = NULL;
p->op_cbfunc = NULL;
p->value_cbfunc = NULL;
p->lookup_cbfunc = NULL;
p->spawn_cbfunc = NULL;
p->cbdata = NULL;
memset(p->nspace, 0, PMIX_MAX_NSLEN+1);
p->rank = -1;
p->key = NULL;
p->value = NULL;
p->procs = NULL;
p->info = NULL;
p->ninfo = 0;
p->nvals = 0;
}
static void cbdes(pmix_cb_t *p)
{
PMIX_DESTRUCT(&p->data);
}
PMIX_CLASS_INSTANCE(pmix_cb_t,
pmix_list_item_t,
cbcon, cbdes);
static void srcon(pmix_usock_sr_t *p)
{
p->peer = NULL;
p->bfr = NULL;
p->cbfunc = NULL;
p->cbdata = NULL;
}
PMIX_CLASS_INSTANCE(pmix_usock_sr_t,
pmix_object_t,
srcon, NULL);
static void pcon(pmix_peer_t *p)
{
p->info = NULL;
p->proc_cnt = 0;
p->server_object = NULL;
p->index = 0;
p->sd = -1;
p->send_ev_active = false;
p->recv_ev_active = false;
PMIX_CONSTRUCT(&p->send_queue, pmix_list_t);
p->send_msg = NULL;
p->recv_msg = NULL;
memset(&p->compat, 0, sizeof(p->compat));
}
static void pdes(pmix_peer_t *p)
{
if (0 <= p->sd) {
CLOSE_THE_SOCKET(p->sd);
}
if (p->send_ev_active) {
event_del(&p->send_event);
}
if (p->recv_ev_active) {
event_del(&p->recv_event);
}
if (NULL != p->info) {
PMIX_RELEASE(p->info);
}
PMIX_LIST_DESTRUCT(&p->send_queue);
if (NULL != p->send_msg) {
PMIX_RELEASE(p->send_msg);
}
if (NULL != p->recv_msg) {
PMIX_RELEASE(p->recv_msg);
}
}
PMIX_CLASS_INSTANCE(pmix_peer_t,
pmix_object_t,
pcon, pdes);
PMIX_CLASS_INSTANCE(pmix_timer_t,
pmix_object_t,
NULL, NULL);

199
opal/mca/pmix/pmix2x/pmix/src/usock/usock.h
Просмотреть файл

@ -1,199 +0,0 @@
/*
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2013 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011 Oak Ridge National Labs. All rights reserved.
* Copyright (c) 2013-2016 Intel, Inc. All rights reserved.
* Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
* All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2016 Mellanox Technologies, Inc.
* All rights reserved.
* Copyright (c) 2016 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*
*/
#ifndef USOCK_H
#define USOCK_H
#include <src/include/pmix_config.h>
#include <src/include/types.h>
#include <pmix_common.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_SYS_UN_H
#include <sys/un.h>
#endif
#ifdef HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif
#ifdef HAVE_NET_UIO_H
#include <net/uio.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include PMIX_EVENT_HEADER
#include "src/include/pmix_globals.h"
#include "src/buffer_ops/buffer_ops.h"
#include "src/class/pmix_hash_table.h"
#include "src/class/pmix_list.h"
/* usock structure for tracking posted recvs */
typedef struct {
pmix_list_item_t super;
pmix_event_t ev;
uint32_t tag;
pmix_usock_cbfunc_t cbfunc;
void *cbdata;
} pmix_usock_posted_recv_t;
PMIX_CLASS_DECLARATION(pmix_usock_posted_recv_t);
/* usock struct for posting send/recv request */
typedef struct {
pmix_object_t super;
pmix_event_t ev;
pmix_peer_t *peer;
pmix_buffer_t *bfr;
pmix_usock_cbfunc_t cbfunc;
void *cbdata;
} pmix_usock_sr_t;
PMIX_CLASS_DECLARATION(pmix_usock_sr_t);
/* usock struct for tracking ops */
typedef struct {
pmix_list_item_t super;
pmix_event_t ev;
volatile bool active;
bool checked;
int status;
pmix_status_t pstatus;
pmix_scope_t scope;
pmix_buffer_t data;
pmix_usock_cbfunc_t cbfunc;
pmix_op_cbfunc_t op_cbfunc;
pmix_value_cbfunc_t value_cbfunc;
pmix_lookup_cbfunc_t lookup_cbfunc;
pmix_spawn_cbfunc_t spawn_cbfunc;
pmix_evhdlr_reg_cbfunc_t errreg_cbfunc;
size_t errhandler_ref;
void *cbdata;
char nspace[PMIX_MAX_NSLEN+1];
pmix_rank_t rank;
char *key;
pmix_value_t *value;
pmix_proc_t *procs;
pmix_info_t *info;
size_t ninfo;
size_t nvals;
} pmix_cb_t;
PMIX_CLASS_DECLARATION(pmix_cb_t);
typedef struct {
pmix_object_t super;
pmix_event_t ev;
void *cbdata;
} pmix_timer_t;
PMIX_CLASS_DECLARATION(pmix_timer_t);
/* internal convenience macros */
#define PMIX_ACTIVATE_SEND_RECV(p, b, cb, d) \
do { \
int rc = -1; \
pmix_usock_sr_t *ms; \
pmix_output_verbose(5, pmix_globals.debug_output, \
"[%s:%d] post send to server", \
__FILE__, __LINE__); \
ms = PMIX_NEW(pmix_usock_sr_t); \
ms->peer = (p); \
ms->bfr = (b); \
ms->cbfunc = (cb); \
ms->cbdata = (d); \
rc = event_assign(&((ms)->ev), pmix_globals.evbase, -1, \
EV_WRITE, pmix_usock_send_recv, (ms)); \
pmix_output_verbose(10, pmix_globals.debug_output, \
"event_assign returned %d", rc); \
event_active(&((ms)->ev), EV_WRITE, 1); \
} while (0)
#define PMIX_ACTIVATE_POST_MSG(ms) \
do { \
pmix_output_verbose(5, pmix_globals.debug_output, \
"[%s:%d] post msg", \
__FILE__, __LINE__); \
event_assign(&((ms)->ev), pmix_globals.evbase, -1, \
EV_WRITE, pmix_usock_process_msg, (ms)); \
event_active(&((ms)->ev), EV_WRITE, 1); \
} while (0)
#define CLOSE_THE_SOCKET(socket) \
do { \
if (0 <= socket) { \
shutdown(socket, 2); \
close(socket); \
socket = -1; \
} \
} while (0)
#define PMIX_TIMER_EVENT(s, f, d) \
do { \
pmix_timer_t *tm; \
struct timeval tv; \
tm = PMIX_NEW(pmix_timer_t); \
tm->cbdata = (d); \
event_assign(&tm->ev, pmix_globals.evbase, -1, 0, (f), tm); \
tv.tv_sec = (s); \
tv.tv_usec = 0; \
PMIX_OUTPUT_VERBOSE((1, pmix_globals.debug_output, \
"defining timer event: %ld sec %ld usec at %s:%d", \
(long)tv.tv_sec, (long)tv.tv_usec, \
__FILE__, __LINE__)); \
event_add(&tm->ev, &tv); \
} while (0)
/* usock common variables */
typedef struct {
pmix_list_t posted_recvs; // list of pmix_usock_posted_recv_t
} pmix_usock_globals_t;
extern pmix_usock_globals_t pmix_usock_globals;
/* usock common functions */
void pmix_usock_init(pmix_usock_cbfunc_t cbfunc);
void pmix_usock_finalize(void);
pmix_status_t pmix_usock_set_nonblocking(int sd);
pmix_status_t pmix_usock_set_blocking(int sd);
pmix_status_t pmix_usock_send_blocking(int sd, char *ptr, size_t size);
pmix_status_t pmix_usock_recv_blocking(int sd, char *data, size_t size);
void pmix_usock_send_recv(int sd, short args, void *cbdata);
void pmix_usock_send_handler(int sd, short flags, void *cbdata);
void pmix_usock_recv_handler(int sd, short flags, void *cbdata);
void pmix_usock_process_msg(int fd, short flags, void *cbdata);
#endif // USOCK_H

6
opal/mca/pmix/pmix2x/pmix/src/util/Makefile.include
Просмотреть файл

@ -46,8 +46,9 @@ headers += \
util/strnlen.h \
util/hash.h \
util/net.h \
util/pif.h \
util/tsd.h \
util/pif.h
util/parse_options.h
sources += \
util/argv.c \
@ -67,7 +68,8 @@ sources += \
util/getid.c \
util/hash.c \
util/net.c \
util/pif.c
util/pif.c \
util/parse_options.c
libpmix_la_LIBADD += \
util/keyval/libpmixutilkeyval.la

4
opal/mca/pmix/pmix2x/pmix/src/util/error.c
Просмотреть файл

@ -145,6 +145,10 @@ const char* PMIx_Error_string(pmix_status_t errnum)
return "PMIX_ERR_VALUE_OUT_OF_BOUNDS";
case PMIX_ERR_NETWORK_NOT_PARSEABLE:
return "PMIX_ERR_NETWORK_NOT_PARSEABLE";
case PMIX_ERR_FILE_OPEN_FAILURE:
return "PMIX_ERR_FILE_OPEN_FAILURE";
case PMIX_ERR_FILE_READ_FAILURE:
return "PMIX_ERR_FILE_READ_FAILURE";
case PMIX_ERR_PERM:
return "PMIX_ERR_PERM";
case PMIX_SUCCESS:

2
opal/mca/pmix/pmix2x/pmix/src/util/error.h
Просмотреть файл

@ -62,6 +62,8 @@
#define PMIX_ERR_PERM (PMIX_INTERNAL_ERR_BASE - 31)
#define PMIX_ERR_OPERATION_IN_PROGRESS (PMIX_INTERNAL_ERR_BASE - 32)
#define PMIX_ERR_NETWORK_NOT_PARSEABLE (PMIX_INTERNAL_ERR_BASE - 33)
#define PMIX_ERR_FILE_OPEN_FAILURE (PMIX_INTERNAL_ERR_BASE - 34)
#define PMIX_ERR_FILE_READ_FAILURE (PMIX_INTERNAL_ERR_BASE - 35)
#define PMIX_ERROR_LOG(r) \
do { \

143
opal/mca/pmix/pmix2x/pmix/src/util/parse_options.c Обычный файл
Просмотреть файл

@ -0,0 +1,143 @@
/*
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "pmix_config.h"
#include "pmix_common.h"
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include "src/util/argv.h"
#include "src/util/output.h"
#include "src/util/parse_options.h"
void pmix_util_parse_range_options(char *inp, char ***output)
{
char **r1=NULL, **r2=NULL;
int i, vint;
int start, end, n;
char nstr[32];
char *input, *bang;
bool bang_option=false;
/* protect against null input */
if (NULL == inp) {
return;
}
/* protect the provided input */
input = strdup(inp);
/* check for the special '!' operator */
if (NULL != (bang = strchr(input, '!'))) {
bang_option = true;
*bang = '\0';
}
/* split on commas */
r1 = pmix_argv_split(input, ',');
/* for each resulting element, check for range */
for (i=0; i < pmix_argv_count(r1); i++) {
r2 = pmix_argv_split(r1[i], '-');
if (1 < pmix_argv_count(r2)) {
/* given range - get start and end */
start = strtol(r2[0], NULL, 10);
end = strtol(r2[1], NULL, 10);
} else {
/* check for wildcard - have to do this here because
* the -1 would have been caught in the split
*/
vint = strtol(r1[i], NULL, 10);
if (-1 == vint) {
pmix_argv_free(*output);
*output = NULL;
pmix_argv_append_nosize(output, "-1");
pmix_argv_free(r2);
goto cleanup;
}
start = strtol(r2[0], NULL, 10);
end = start;
}
for (n = start; n <= end; n++) {
snprintf(nstr, 32, "%d", n);
pmix_argv_append_nosize(output, nstr);
}
pmix_argv_free(r2);
}
cleanup:
if (bang_option) {
pmix_argv_append_nosize(output, "BANG");
}
free(input);
pmix_argv_free(r1);
}
void pmix_util_get_ranges(char *inp, char ***startpts, char ***endpts)
{
char **r1=NULL, **r2=NULL;
int i;
char *input;
/* protect against null input */
if (NULL == inp) {
return;
}
/* protect the provided input */
input = strdup(inp);
/* split on commas */
r1 = pmix_argv_split(input, ',');
/* for each resulting element, check for range */
for (i=0; i < pmix_argv_count(r1); i++) {
r2 = pmix_argv_split(r1[i], '-');
if (2 == pmix_argv_count(r2)) {
/* given range - get start and end */
pmix_argv_append_nosize(startpts, r2[0]);
pmix_argv_append_nosize(endpts, r2[1]);
} else if (1 == pmix_argv_count(r2)) {
/* only one value provided, so it is both the start
* and the end
*/
pmix_argv_append_nosize(startpts, r2[0]);
pmix_argv_append_nosize(endpts, r2[0]);
} else {
/* no idea how to parse this */
pmix_output(0, "Unknown parse error on string: %s(%s)", inp, r1[i]);
}
pmix_argv_free(r2);
}
free(input);
pmix_argv_free(r1);
}

37
opal/mca/pmix/pmix2x/pmix/src/util/parse_options.h Обычный файл
Просмотреть файл

@ -0,0 +1,37 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/** @file:
*
*/
#ifndef _PMIX_PARSE_OPTIONS_H_
#define _PMIX_PARSE_OPTIONS_H_
#include "pmix_config.h"
BEGIN_C_DECLS
PMIX_EXPORT void pmix_util_parse_range_options(char *input, char ***output);
PMIX_EXPORT void pmix_util_get_ranges(char *inp, char ***startpts, char ***endpts);
END_C_DECLS
#endif

5
opal/mca/pmix/pmix2x/pmix/src/util/pif.c
Просмотреть файл

@ -269,10 +269,11 @@ int16_t pmix_ifaddrtokindex(const char* if_addr)
for (r = res; r != NULL; r = r->ai_next) {
PMIX_LIST_FOREACH(intf, &pmix_if_list, pmix_pif_t) {
if (AF_INET == r->ai_family && AF_INET == intf->af_family) {
struct sockaddr_in ipv4;
struct sockaddr ipv4;
memset(&ipv4, 0, sizeof(struct sockaddr));
len = (r->ai_addrlen < sizeof(struct sockaddr_in)) ? r->ai_addrlen : sizeof(struct sockaddr_in);
memcpy(&ipv4, r->ai_addr, len);
if (pmix_net_samenetwork((struct sockaddr*)&ipv4, (struct sockaddr*)&intf->if_addr, intf->if_mask)) {
if (pmix_net_samenetwork(&ipv4, (struct sockaddr*)&intf->if_addr, intf->if_mask)) {
if_kernel_index = intf->if_kernel_index;
freeaddrinfo (res);
return if_kernel_index;

4
opal/mca/pmix/pmix2x/pmix/src/util/strnlen.h
Просмотреть файл

@ -19,11 +19,11 @@
#if defined(HAVE_STRNLEN)
#define PMIX_STRNLEN(c, a, b) \
(c) = (int)strnlen(a, b)
(c) = strnlen(a, b)
#else
#define PMIX_STRNLEN(c, a, b) \
do { \
int _x; \
size_t _x; \
(c) = 0; \
for (_x=0; _x < (b); _x++) { \
if ('\0' == (a)[_x]) { \

2
opal/mca/pmix/pmix2x/pmix/test/cli_stages.c
Просмотреть файл

@ -60,7 +60,7 @@ void cli_connect(cli_info_t *cli, int sd, struct event_base * ebase, event_callb
cli->ev = event_new(ebase, sd,
EV_READ|EV_PERSIST, callback, cli);
event_add(cli->ev,NULL);
pmix_usock_set_nonblocking(sd);
pmix_ptl_base_set_nonblocking(sd);
TEST_VERBOSE(("Connection accepted from rank %d", cli_rank(cli) ));
cli->state = CLI_CONNECTED;
}

2
opal/mca/pmix/pmix2x/pmix/test/cli_stages.h
Просмотреть файл

@ -25,7 +25,7 @@
#include "src/include/pmix_globals.h"
#include "pmix_server.h"
#include "src/class/pmix_list.h"
#include "src/usock/usock.h"
#include "src/mca/ptl/base/base.h"
#include "test_common.h"

0
opal/mca/pmix/pmix2x/pmix/test/pmix_client_otheruser.sh Обычный файл → Исполняемый файл
Просмотреть файл

8
opal/mca/pmix/pmix2x/pmix/test/simple/simpdmodex.c
Просмотреть файл

@ -35,12 +35,18 @@
#include "src/buffer_ops/types.h"
#include "src/util/output.h"
#include "src/util/printf.h"
#include "src/usock/usock.h"
static uint32_t nprocs;
static pmix_proc_t myproc;
static uint32_t getcount = 0;
#define PMIX_WAIT_FOR_COMPLETION(a) \
do { \
while ((a)) { \
usleep(10); \
} \
} while (0)
static void opcbfunc(pmix_status_t status, void *cbdata)
{
bool *active = (bool*)cbdata;

92
opal/mca/pmix/pmix2x/pmix/test/simple/simptest.c
Просмотреть файл

@ -44,7 +44,6 @@
#include "src/util/printf.h"
#include "src/util/argv.h"
#include "src/buffer_ops/buffer_ops.h"
#include "src/usock/usock.h"
static pmix_status_t connected(const pmix_proc_t *proc, void *server_object,
pmix_op_cbfunc_t cbfunc, void *cbdata);
@ -209,7 +208,9 @@ int main(int argc, char **argv)
myxfer_t *x;
pmix_proc_t proc;
wait_tracker_t *child;
pmix_info_t info;
pmix_info_t info[2];
bool cross_version = false;
bool usock = true;
/* smoke test */
if (PMIX_SUCCESS != 0) {
@ -219,14 +220,57 @@ int main(int argc, char **argv)
fprintf(stderr, "Testing version %s\n", PMIx_Get_version());
/* see if we were passed the number of procs to run or
* the executable to use */
for (n=1; n < argc; n++) {
if (0 == strcmp("-n", argv[n]) &&
NULL != argv[n+1]) {
nprocs = strtol(argv[n+1], NULL, 10);
++n; // step over the argument
} else if (0 == strcmp("-e", argv[n]) &&
NULL != argv[n+1]) {
executable = strdup(argv[n+1]);
for (k=n+2; NULL != argv[k]; k++) {
pmix_argv_append_nosize(&client_argv, argv[k]);
}
n += k;
} else if (0 == strcmp("-x", argv[n])) {
/* cross-version test - we will set one child to
* run at a different version. Requires -n >= 2 */
cross_version = true;
usock = false;
} else if (0 == strcmp("-u", argv[n])) {
/* enable usock */
usock = false;
} else if (0 == strcmp("-h", argv[n])) {
/* print the options and exit */
fprintf(stderr, "usage: simptest <options>\n");
fprintf(stderr, " -n N Number of clients to run\n");
fprintf(stderr, " -e foo Name of the client executable to run (default: simpclient\n");
fprintf(stderr, " -x Test cross-version support\n");
fprintf(stderr, " -u Enable legacy usock support\n");
exit(0);
}
}
if (NULL == executable) {
executable = strdup("./simpclient");
}
if (cross_version && nprocs < 2) {
fprintf(stderr, "Cross-version testing requires at least two clients\n");
exit(1);
}
/* setup the server library and tell it to support tool connections */
PMIX_INFO_CONSTRUCT(&info);
(void)strncpy(info.key, PMIX_SERVER_TOOL_SUPPORT, PMIX_MAX_KEYLEN);
if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, &info, 1))) {
PMIX_INFO_CONSTRUCT(&info[0]);
(void)strncpy(info[0].key, PMIX_SERVER_TOOL_SUPPORT, PMIX_MAX_KEYLEN);
PMIX_INFO_CONSTRUCT(&info[1]);
PMIX_INFO_LOAD(&info[1], PMIX_USOCK_DISABLE, &usock, PMIX_BOOL);
if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, 2))) {
fprintf(stderr, "Init failed with error %d\n", rc);
return rc;
}
PMIX_INFO_DESTRUCT(&info);
PMIX_INFO_DESTRUCT(&info[0]);
PMIX_INFO_DESTRUCT(&info[1]);
/* register the errhandler */
PMIx_Register_event_handler(NULL, 0, NULL, 0,
@ -241,26 +285,6 @@ int main(int argc, char **argv)
EV_SIGNAL|EV_PERSIST,wait_signal_callback, &handler);
event_add(&handler, NULL);
/* see if we were passed the number of procs to run or
* the executable to use */
for (n=1; n < (argc-1); n++) {
if (0 == strcmp("-n", argv[n]) &&
NULL != argv[n+1]) {
nprocs = strtol(argv[n+1], NULL, 10);
++n; // step over the argument
} else if (0 == strcmp("-e", argv[n]) &&
NULL != argv[n+1]) {
executable = strdup(argv[n+1]);
for (k=n+2; NULL != argv[k]; k++) {
pmix_argv_append_nosize(&client_argv, argv[k]);
}
n += k;
}
}
if (NULL == executable) {
executable = strdup("./simpclient");
}
/* we have a single namespace for all clients */
atmp = NULL;
for (n=0; n < nprocs; n++) {
@ -296,10 +320,19 @@ int main(int argc, char **argv)
PMIx_server_finalize();
return rc;
}
/* if cross-version test is requested, then oscillate PTL support
* by rank */
if (cross_version) {
if (0 == n % 2) {
pmix_setenv("PMIX_MCA_ptl", "tcp", true, &client_env);
} else {
pmix_setenv("PMIX_MCA_ptl", "usock", true, &client_env);
}
}
x = PMIX_NEW(myxfer_t);
if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid,
NULL, opcbfunc, x))) {
fprintf(stderr, "Server fork setup failed with error %d\n", rc);
fprintf(stderr, "Server register client failed with error %d\n", rc);
PMIx_server_finalize();
return rc;
}
@ -425,9 +458,8 @@ static pmix_status_t connected(const pmix_proc_t *proc, void *server_object,
static pmix_status_t finalized(const pmix_proc_t *proc, void *server_object,
pmix_op_cbfunc_t cbfunc, void *cbdata)
{
pmix_output(0, "SERVER: FINALIZED %s:%d",
proc->nspace, proc->rank);
--wakeup;
pmix_output(0, "SERVER: FINALIZED %s:%d WAKEUP %d",
proc->nspace, proc->rank, wakeup);
/* ensure we call the cbfunc so the proc can exit! */
if (NULL != cbfunc) {
cbfunc(PMIX_SUCCESS, cbdata);

7
opal/mca/pmix/pmix2x/pmix/test/test_cd.c
Просмотреть файл

@ -16,6 +16,13 @@ typedef struct {
int status;
} cd_cbdata;
#define PMIX_WAIT_FOR_COMPLETION(a) \
do { \
while ((a)) { \
usleep(10); \
} \
} while (0)
static void cd_cb(pmix_status_t status, void *cbdata)
{
cd_cbdata *cb = (cd_cbdata*)cbdata;

1
opal/mca/pmix/pmix2x/pmix/test/test_common.h
Просмотреть файл

@ -28,7 +28,6 @@
#include "src/class/pmix_list.h"
#include "src/util/argv.h"
#include "src/usock/usock.h"
#define TEST_NAMESPACE "smoky_nspace"
#define TEST_CREDENTIAL "dummy"

7
opal/mca/pmix/pmix2x/pmix/test/test_spawn.c
Просмотреть файл

@ -18,6 +18,13 @@ typedef struct {
char nspace[PMIX_MAX_NSLEN];
} spawn_cbdata;
#define PMIX_WAIT_FOR_COMPLETION(a) \
do { \
while ((a)) { \
usleep(10); \
} \
} while (0)
static void spawn_cb(pmix_status_t status,
char nspace[], void *cbdata)
{

12
opal/mca/pmix/pmix_types.h
Просмотреть файл

@ -62,8 +62,18 @@ BEGIN_C_DECLS
#define OPAL_PMIX_GRPID "pmix.egid" // (uint32_t) effective group id
/* attributes for the rendezvous socket */
#define OPAL_PMIX_USOCK_DISABLE "pmix.usock.disable" // (bool) disable legacy usock support
#define OPAL_PMIX_SOCKET_MODE "pmix.sockmode" // (uint32_t) POSIX mode_t (9 bits valid)
/* attributes for TCP connections */
#define OPAL_PMIX_TCP_URI "pmix.tcp.uri" // (char*) URI of server to connect to
#define OPAL_PMIX_TCP_IF_INCLUDE "pmix.tcp.ifinclude" // (char*) comma-delimited list of devices and/or CIDR notation
#define OPAL_PMIX_TCP_IF_EXCLUDE "pmix.tcp.ifexclude" // (char*) comma-delimited list of devices and/or CIDR notation
#define OPAL_PMIX_TCP_IPV4_PORT "pmix.tcp.ipv4" // (int) IPv4 port to be used
#define OPAL_PMIX_TCP_IPV6_PORT "pmix.tcp.ipv6" // (int) IPv6 port to be used
#define OPAL_PMIX_TCP_DISABLE_IPV4 "pmix.tcp.disipv4" // (bool) true to disable IPv4 family
#define OPAL_PMIX_TCP_DISABLE_IPV6 "pmix.tcp.disipv6" // (bool) true to disable IPv6 family
/* general proc-level attributes */
#define OPAL_PMIX_CPUSET "pmix.cpuset" // (char*) hwloc bitmap applied to proc upon launch
#define OPAL_PMIX_CREDENTIAL "pmix.cred" // (char*) security credential assigned to proc
@ -123,6 +133,8 @@ BEGIN_C_DECLS
/* request-related info */
#define OPAL_PMIX_COLLECT_DATA "pmix.collect" // (bool) collect data and return it at the end of the operation
#define OPAL_PMIX_TIMEOUT "pmix.timeout" // (int) time in sec before specified operation should time out
#define OPAL_PMIX_IMMEDIATE "pmix.immediate" // (bool) specified operation should immediately return an error if requested
// data cannot be found - do not request it from the host RM
#define OPAL_PMIX_WAIT "pmix.wait" // (int) caller requests that the server wait until at least the specified
// #values are found (0 => all and is the default)
#define OPAL_PMIX_COLLECTIVE_ALGO "pmix.calgo" // (char*) comma-delimited list of algorithms to use for collective

2
orte/mca/ess/singleton/ess_singleton_module.c
Просмотреть файл

@ -633,7 +633,7 @@ static int fork_hnp(void)
orte_process_info.my_hnp_uri = orted_uri;
/* split the pmix_uri into its parts */
argv = opal_argv_split(cptr, ',');
argv = opal_argv_split(cptr, '*');
count = opal_argv_count(argv);
/* push each piece into the environment */
for (i=0; i < count; i++) {

5
orte/orted/orted_main.c
Просмотреть файл

@ -612,12 +612,13 @@ int orte_daemon(int argc, char *argv[])
opal_argv_append_nosize(&singenv, env_str);
free(env_str);
nptr = opal_argv_join(singenv, ',');
nptr = opal_argv_join(singenv, '*');
opal_argv_free(singenv);
/* create a string that contains our uri + sysinfo + PMIx server URI envars */
orte_util_convert_sysinfo_to_string(&sysinfo, orte_local_cpu_type, orte_local_cpu_model);
asprintf(&tmp, "%s[%s]%s", orte_process_info.my_daemon_uri, sysinfo, nptr);
free(sysinfo);
free(sysinfo);
free(nptr);
/* pass that info to the singleton */

6
orte/orted/pmix/pmix_server.c
Просмотреть файл

@ -272,6 +272,12 @@ int pmix_server_init(void)
kv->type = OPAL_STRING;
kv->data.string = strdup(orte_process_info.tmpdir_base);
opal_list_append(&info, &kv->super);
/* disable usock */
kv = OBJ_NEW(opal_value_t);
kv->key = strdup(OPAL_PMIX_USOCK_DISABLE);
kv->type = OPAL_BOOL;
kv->data.flag = true;
opal_list_append(&info, &kv->super);
/* setup the local server */
if (ORTE_SUCCESS != (rc = opal_pmix.server_init(&pmix_server, &info))) {