552c9ca5a0
WHAT: Open our low-level communication infrastructure by moving all necessary components (btl/rcache/allocator/mpool) down in OPAL All the components required for inter-process communications are currently deeply integrated in the OMPI layer. Several groups/institutions have express interest in having a more generic communication infrastructure, without all the OMPI layer dependencies. This communication layer should be made available at a different software level, available to all layers in the Open MPI software stack. As an example, our ORTE layer could replace the current OOB and instead use the BTL directly, gaining access to more reactive network interfaces than TCP. Similarly, external software libraries could take advantage of our highly optimized AM (active message) communication layer for their own purpose. UTK with support from Sandia, developped a version of Open MPI where the entire communication infrastucture has been moved down to OPAL (btl/rcache/allocator/mpool). Most of the moved components have been updated to match the new schema, with few exceptions (mainly BTLs where I have no way of compiling/testing them). Thus, the completion of this RFC is tied to being able to completing this move for all BTLs. For this we need help from the rest of the Open MPI community, especially those supporting some of the BTLs. A non-exhaustive list of BTLs that qualify here is: mx, portals4, scif, udapl, ugni, usnic. This commit was SVN r32317.
261 строка
7.8 KiB
C
261 строка
7.8 KiB
C
/*
|
|
* Copyright (c) 2014 Cisco Systems, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "opal_config.h"
|
|
|
|
#include <assert.h>
|
|
#include <sys/stat.h>
|
|
#include <sys/socket.h>
|
|
#include <sys/un.h>
|
|
#include <sys/types.h>
|
|
#include <string.h>
|
|
#include <unistd.h>
|
|
#include <alloca.h>
|
|
#include <time.h>
|
|
|
|
#include "opal_stdint.h"
|
|
#include "opal/threads/mutex.h"
|
|
#include "opal/mca/event/event.h"
|
|
#include "opal/mca/dstore/dstore.h"
|
|
#include "opal/util/output.h"
|
|
#include "opal/util/fd.h"
|
|
|
|
#include "ompi/mca/rte/rte.h"
|
|
|
|
#include "btl_usnic.h"
|
|
#include "btl_usnic_module.h"
|
|
#include "btl_usnic_connectivity.h"
|
|
|
|
/**************************************************************************
|
|
* Client-side data and methods
|
|
**************************************************************************/
|
|
|
|
static bool initialized = false;
|
|
static int agent_fd = -1;
|
|
|
|
|
|
/*
|
|
* Startup the agent and share our MCA param values with the it.
|
|
*/
|
|
int opal_btl_usnic_connectivity_client_init(void)
|
|
{
|
|
/* If connectivity checking is not enabled, do nothing */
|
|
if (!mca_btl_usnic_component.connectivity_enabled) {
|
|
return OPAL_SUCCESS;
|
|
}
|
|
assert(!initialized);
|
|
|
|
/* Open local IPC socket to the agent */
|
|
agent_fd = socket(PF_UNIX, SOCK_STREAM, 0);
|
|
if (agent_fd < 0) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("socket() failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
char *ipc_filename = NULL;
|
|
asprintf(&ipc_filename, "%s/%s",
|
|
ompi_process_info.job_session_dir, CONNECTIVITY_SOCK_NAME);
|
|
if (NULL == ipc_filename) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("Out of memory");
|
|
/* Will not return */
|
|
}
|
|
#if !defined(NDEBUG)
|
|
struct sockaddr_un sun;
|
|
assert(strlen(ipc_filename) <= sizeof(sun.sun_path));
|
|
#endif
|
|
|
|
/* Wait for the agent to create its socket. Timeout after 10
|
|
seconds if we don't find the socket. */
|
|
struct stat sbuf;
|
|
time_t start = time(NULL);
|
|
while (1) {
|
|
int ret = stat(ipc_filename, &sbuf);
|
|
if (0 == ret) {
|
|
break;
|
|
} else if (ENOENT != errno) {
|
|
/* If the error wasn't "file not found", then something
|
|
else Bad happened */
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("stat() failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
/* If the named socket wasn't there yet, then give the agent a
|
|
little time to establish it */
|
|
usleep(1);
|
|
|
|
if (time(NULL) - start > 10) {
|
|
ABORT("connectivity client timeout waiting for server socket to appear");
|
|
/* Will not return */
|
|
}
|
|
}
|
|
|
|
/* Connect */
|
|
struct sockaddr_un address;
|
|
memset(&address, 0, sizeof(struct sockaddr_un));
|
|
address.sun_family = AF_UNIX;
|
|
strncpy(address.sun_path, ipc_filename, sizeof(address.sun_path));
|
|
|
|
if (0 != connect(agent_fd, (struct sockaddr*) &address, sizeof(address))) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("connect() failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
/* Send the magic token */
|
|
int tlen = strlen(CONNECTIVITY_MAGIC_TOKEN);
|
|
if (OPAL_SUCCESS != opal_fd_write(agent_fd, tlen,
|
|
CONNECTIVITY_MAGIC_TOKEN)) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("usnic connectivity client IPC connect write failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
/* Receive a magic token back */
|
|
char *ack = alloca(tlen + 1);
|
|
if (NULL == ack) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("Out of memory");
|
|
/* Will not return */
|
|
}
|
|
if (OPAL_SUCCESS != opal_fd_read(agent_fd, tlen, ack)) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("usnic connectivity client IPC connect read failed");
|
|
/* Will not return */
|
|
}
|
|
if (memcmp(ack, CONNECTIVITY_MAGIC_TOKEN, tlen) != 0) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("usnic connectivity client got wrong token back from agent");
|
|
/* Will not return */
|
|
}
|
|
|
|
/* All done */
|
|
initialized = true;
|
|
opal_output_verbose(20, USNIC_OUT,
|
|
"usNIC connectivity client initialized");
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
|
|
/*
|
|
* Send a listen command to the agent
|
|
*/
|
|
int opal_btl_usnic_connectivity_listen(opal_btl_usnic_module_t *module)
|
|
{
|
|
/* If connectivity checking is not enabled, do nothing */
|
|
if (!mca_btl_usnic_component.connectivity_enabled) {
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
/* Send the LISTEN command */
|
|
int id = CONNECTIVITY_AGENT_CMD_LISTEN;
|
|
if (OPAL_SUCCESS != opal_fd_write(agent_fd, sizeof(id), &id)) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("usnic connectivity client IPC write failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
/* Send the LISTEN command parameters */
|
|
opal_btl_usnic_connectivity_cmd_listen_t cmd = {
|
|
.ipv4_addr = module->local_addr.ipv4_addr,
|
|
.cidrmask = module->local_addr.cidrmask,
|
|
.mtu = module->local_addr.mtu
|
|
};
|
|
/* Ensure to NULL-terminate the passed strings */
|
|
strncpy(cmd.nodename, ompi_process_info.nodename,
|
|
CONNECTIVITY_NODENAME_LEN - 1);
|
|
strncpy(cmd.if_name, module->if_name, CONNECTIVITY_IFNAME_LEN - 1);
|
|
strncpy(cmd.usnic_name, ibv_get_device_name(module->device),
|
|
CONNECTIVITY_IFNAME_LEN - 1);
|
|
memcpy(cmd.mac, module->local_addr.mac, 6);
|
|
|
|
if (OPAL_SUCCESS != opal_fd_write(agent_fd, sizeof(cmd), &cmd)) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("usnic connectivity client IPC write failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
/* Wait for the reply with the UDP port */
|
|
opal_btl_usnic_connectivity_cmd_listen_reply_t reply;
|
|
memset(&reply, 0, sizeof(reply));
|
|
if (OPAL_SUCCESS != opal_fd_read(agent_fd, sizeof(reply), &reply)) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("usnic connectivity client IPC read failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
/* Get the UDP port number that was received */
|
|
assert(CONNECTIVITY_AGENT_CMD_LISTEN == reply.cmd);
|
|
module->local_addr.connectivity_udp_port = reply.udp_port;
|
|
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
|
|
int opal_btl_usnic_connectivity_ping(uint32_t src_ipv4_addr, int src_port,
|
|
uint32_t dest_ipv4_addr,
|
|
uint32_t dest_cidrmask, int dest_port,
|
|
uint8_t dest_mac[6], char *dest_nodename,
|
|
size_t mtu)
|
|
{
|
|
/* If connectivity checking is not enabled, do nothing */
|
|
if (!mca_btl_usnic_component.connectivity_enabled) {
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
/* Send the PING command */
|
|
int id = CONNECTIVITY_AGENT_CMD_PING;
|
|
if (OPAL_SUCCESS != opal_fd_write(agent_fd, sizeof(id), &id)) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("usnic connectivity client IPC write failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
/* Send the PING command parameters */
|
|
opal_btl_usnic_connectivity_cmd_ping_t cmd = {
|
|
.src_ipv4_addr = src_ipv4_addr,
|
|
.src_udp_port = src_port,
|
|
.dest_ipv4_addr = dest_ipv4_addr,
|
|
.dest_cidrmask = dest_cidrmask,
|
|
.dest_udp_port = dest_port,
|
|
.mtu = mtu
|
|
};
|
|
/* Ensure to NULL-terminate the passed string */
|
|
strncpy(cmd.dest_nodename, dest_nodename, CONNECTIVITY_NODENAME_LEN - 1);
|
|
memcpy(cmd.dest_mac, dest_mac, 6);
|
|
|
|
if (OPAL_SUCCESS != opal_fd_write(agent_fd, sizeof(cmd), &cmd)) {
|
|
OPAL_ERROR_LOG(OPAL_ERR_IN_ERRNO);
|
|
ABORT("usnic connectivity client IPC write failed");
|
|
/* Will not return */
|
|
}
|
|
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
|
|
/*
|
|
* Shut down the connectivity client
|
|
*/
|
|
int opal_btl_usnic_connectivity_client_finalize(void)
|
|
{
|
|
/* Make it safe to finalize, even if we weren't initialized */
|
|
if (!initialized) {
|
|
return OPAL_SUCCESS;
|
|
}
|
|
|
|
close(agent_fd);
|
|
agent_fd = -1;
|
|
|
|
initialized = false;
|
|
return OPAL_SUCCESS;
|
|
}
|