780c93ee57
We recognize that this means other users of OPAL will need to "wrap" the opal_process_name_t if they desire to abstract it in some fashion. This is regrettable, and we are looking at possible alternatives that might mitigate that requirement. Meantime, however, we have to put the needs of the OMPI community first, and are taking this step to restore hetero and SPARC support.
221 строка
8.5 KiB
C
221 строка
8.5 KiB
C
/*
|
|
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2006-2013 Los Alamos National Security, LLC.
|
|
* All rights reserved.
|
|
* Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
|
|
* Copyright (c) 2013-2014 Intel, Inc. All rights reserved.
|
|
* Copyright (c) 2014 Mellanox Technologies, Inc.
|
|
* All rights reserved.
|
|
* Copyright (c) 2014 Research Organization for Information Science
|
|
* and Technology (RIST). All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#ifndef _PMIX_SERVER_INTERNAL_H_
|
|
#define _PMIX_SERVER_INTERNAL_H_
|
|
|
|
#include "orte_config.h"
|
|
#include "orte/types.h"
|
|
|
|
#ifdef HAVE_SYS_SOCKET_H
|
|
#include <sys/socket.h>
|
|
#endif
|
|
#ifdef HAVE_SYS_UN_H
|
|
#include <sys/un.h>
|
|
#endif
|
|
|
|
#include "opal/types.h"
|
|
#include "opal/mca/base/base.h"
|
|
#include "opal/mca/event/event.h"
|
|
#include "opal/mca/pmix/pmix.h"
|
|
#include "opal/util/proc.h"
|
|
|
|
BEGIN_C_DECLS
|
|
|
|
/**
|
|
* the state of the connection
|
|
*/
|
|
typedef enum {
|
|
PMIX_SERVER_UNCONNECTED,
|
|
PMIX_SERVER_CLOSED,
|
|
PMIX_SERVER_RESOLVE,
|
|
PMIX_SERVER_CONNECTING,
|
|
PMIX_SERVER_CONNECT_ACK,
|
|
PMIX_SERVER_CONNECTED,
|
|
PMIX_SERVER_FAILED,
|
|
PMIX_SERVER_ACCEPTING
|
|
} pmix_server_state_t;
|
|
|
|
/* define a command type for client-server communications */
|
|
typedef uint8_t pmix_cmd_t;
|
|
#define PMIX_CMD_T OPAL_UINT8
|
|
|
|
/* define some commands */
|
|
#define PMIX_ABORT_CMD 1
|
|
#define PMIX_FENCE_CMD 2
|
|
#define PMIX_FENCENB_CMD 3
|
|
#define PMIX_PUT_CMD 4
|
|
#define PMIX_GET_CMD 5
|
|
#define PMIX_GETNB_CMD 6
|
|
#define PMIX_FINALIZE_CMD 7
|
|
#define PMIX_GETATTR_CMD 8
|
|
|
|
/* define some message types */
|
|
#define PMIX_USOCK_IDENT 1
|
|
#define PMIX_USOCK_USER 2
|
|
|
|
/* header for pmix client-server msgs - must
|
|
* match that in opal/mca/pmix/native! */
|
|
typedef struct {
|
|
opal_process_name_t id;
|
|
uint8_t type;
|
|
uint32_t tag;
|
|
size_t nbytes;
|
|
} pmix_server_hdr_t;
|
|
|
|
/* usock structure for sending a message */
|
|
typedef struct {
|
|
opal_list_item_t super;
|
|
pmix_server_hdr_t hdr;
|
|
opal_buffer_t *data;
|
|
bool hdr_sent;
|
|
char *sdptr;
|
|
size_t sdbytes;
|
|
} pmix_server_send_t;
|
|
OBJ_CLASS_DECLARATION(pmix_server_send_t);
|
|
|
|
/* usock structure for recving a message */
|
|
typedef struct {
|
|
opal_list_item_t super;
|
|
pmix_server_hdr_t hdr;
|
|
bool hdr_recvd;
|
|
char *data;
|
|
char *rdptr;
|
|
size_t rdbytes;
|
|
} pmix_server_recv_t;
|
|
OBJ_CLASS_DECLARATION(pmix_server_recv_t);
|
|
|
|
/* object for tracking peers - each peer can have multiple
|
|
* connections. This can occur if the initial app executes
|
|
* a fork/exec, and the child initiates its own connection
|
|
* back to the PMIx server. Thus, the trackers are "indexed"
|
|
* by the socket, not the process name */
|
|
typedef struct {
|
|
opal_object_t super;
|
|
int sd;
|
|
orte_process_name_t name;
|
|
int retries; // number of times we have tried to connect to this address
|
|
pmix_server_state_t state;
|
|
opal_event_t op_event; // used for connecting and operations other than read/write
|
|
opal_event_t send_event; /**< registration with event thread for send events */
|
|
bool send_ev_active;
|
|
opal_event_t recv_event; /**< registration with event thread for recv events */
|
|
bool recv_ev_active;
|
|
opal_event_t timer_event; /**< timer for retrying connection failures */
|
|
bool timer_ev_active;
|
|
opal_list_t send_queue; /**< list of messages to send */
|
|
pmix_server_send_t *send_msg; /**< current send in progress */
|
|
pmix_server_recv_t *recv_msg; /**< current recv in progress */
|
|
} pmix_server_peer_t;
|
|
OBJ_CLASS_DECLARATION(pmix_server_peer_t);
|
|
|
|
/* object for tracking remote modex requests so we can
|
|
* correctly route the eventual reply */
|
|
typedef struct {
|
|
opal_list_item_t super;
|
|
pmix_server_peer_t *peer;
|
|
orte_proc_t *proxy;
|
|
opal_process_name_t target;
|
|
uint32_t tag;
|
|
} pmix_server_dmx_req_t;
|
|
OBJ_CLASS_DECLARATION(pmix_server_dmx_req_t);
|
|
|
|
/* queue a message to be sent by one of our procs - must
|
|
* provide the following params:
|
|
*
|
|
* p - the peer object of the process
|
|
* t - tag to be sent to
|
|
* b - buffer to be sent
|
|
*/
|
|
#define PMIX_SERVER_QUEUE_SEND(p, t, b) \
|
|
do { \
|
|
pmix_server_send_t *msg; \
|
|
opal_output_verbose(2, pmix_server_output, \
|
|
"%s:[%s:%d] queue send to %s", \
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), \
|
|
__FILE__, __LINE__, \
|
|
ORTE_NAME_PRINT(&(p)->name)); \
|
|
msg = OBJ_NEW(pmix_server_send_t); \
|
|
/* setup the header */ \
|
|
msg->hdr.id = OPAL_PROC_MY_NAME; \
|
|
msg->hdr.type = PMIX_USOCK_USER; \
|
|
msg->hdr.tag = (t); \
|
|
msg->hdr.nbytes = (b)->bytes_used; \
|
|
/* point to the buffer */ \
|
|
msg->data = (b); \
|
|
/* start the send with the header */ \
|
|
msg->sdptr = (char*)&msg->hdr; \
|
|
msg->sdbytes = sizeof(pmix_server_hdr_t); \
|
|
/* if there is no message on-deck, put this one there */ \
|
|
if (NULL == (p)->send_msg) { \
|
|
(p)->send_msg = msg; \
|
|
} else { \
|
|
/* add it to the queue */ \
|
|
opal_list_append(&(p)->send_queue, &msg->super); \
|
|
} \
|
|
/* ensure the send event is active */ \
|
|
if (!(p)->send_ev_active) { \
|
|
opal_event_add(&(p)->send_event, 0); \
|
|
(p)->send_ev_active = true; \
|
|
} \
|
|
}while(0);
|
|
|
|
#define CLOSE_THE_SOCKET(socket) \
|
|
do { \
|
|
shutdown(socket, 2); \
|
|
close(socket); \
|
|
} while(0)
|
|
|
|
/* expose shared functions */
|
|
extern void pmix_server_send_handler(int fd, short args, void *cbdata);
|
|
extern void pmix_server_recv_handler(int fd, short args, void *cbdata);
|
|
extern int pmix_server_peer_recv_connect_ack(pmix_server_peer_t* pr,
|
|
int sd, pmix_server_hdr_t *dhdr);
|
|
extern void pmix_server_recv_handler(int sd, short flags, void *cbdata);
|
|
extern void pmix_server_peer_connected(pmix_server_peer_t* peer);
|
|
extern int pmix_server_send_connect_ack(pmix_server_peer_t* peer);
|
|
extern int pmix_server_recv_connect_ack(pmix_server_peer_t* pr, int sd,
|
|
pmix_server_hdr_t *dhdr);
|
|
extern void pmix_server_peer_event_init(pmix_server_peer_t* peer);
|
|
extern char* pmix_server_state_print(pmix_server_state_t state);
|
|
extern pmix_server_peer_t* pmix_server_peer_lookup(int sd);
|
|
extern void pmix_server_peer_dump(pmix_server_peer_t* peer, const char* msg);
|
|
extern int pack_segment_info(opal_process_name_t id, opal_buffer_t *reply);
|
|
|
|
|
|
/* exposed shared variables */
|
|
extern bool pmix_server_distribute_data;
|
|
extern opal_hash_table_t *pmix_server_peers;
|
|
extern int pmix_server_verbosity;
|
|
extern int pmix_server_output;
|
|
extern int pmix_server_local_handle, pmix_server_remote_handle, pmix_server_global_handle;
|
|
extern opal_list_t pmix_server_pending_dmx_reqs;
|
|
|
|
END_C_DECLS
|
|
|
|
#endif /* PMIX_SERVER_INTERNAL_H_ */
|
|
|