628125599d
when running an MPI job spanning a node that has two TCP NICs and a node that has one TCP NIC. Previously, for the 2 NIC/module process, we would return the first peer IP address if we couldn't find a subnet match with any of the peer's published IP addresses -- this was to support running OMPI across subnet boundaries. Changed the behavior to only do that behavior if the IP address we're trying to match is public (i.e., not 10.x.y.z, 192.168.x.y, or 172.16.x.y) *and* any of the remote peer's addresses are public (working on the assumption that if we both have public addresses, they're routable to each other). This definitely will not work in all scenarios, such as when we go to WAN kinds of executions, and will need to be revisited at that time. This commit was SVN r9119.
336 строки
12 KiB
C
336 строки
12 KiB
C
/*
|
|
* Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "ompi_config.h"
|
|
|
|
#ifdef HAVE_ARPA_INET_H
|
|
#include <arpa/inet.h>
|
|
#endif
|
|
#ifdef HAVE_NETINET_IN_H
|
|
#include <netinet/in.h>
|
|
#endif
|
|
|
|
#include "orte/class/orte_proc_table.h"
|
|
#include "ompi/mca/btl/base/btl_base_error.h"
|
|
#include "ompi/mca/pml/base/pml_base_module_exchange.h"
|
|
|
|
#include "btl_tcp.h"
|
|
#include "btl_tcp_proc.h"
|
|
|
|
static void mca_btl_tcp_proc_construct(mca_btl_tcp_proc_t* proc);
|
|
static void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t* proc);
|
|
static bool is_private_ipv4(struct in_addr *in);
|
|
|
|
|
|
OBJ_CLASS_INSTANCE(
|
|
mca_btl_tcp_proc_t,
|
|
opal_list_item_t,
|
|
mca_btl_tcp_proc_construct,
|
|
mca_btl_tcp_proc_destruct);
|
|
|
|
|
|
void mca_btl_tcp_proc_construct(mca_btl_tcp_proc_t* proc)
|
|
{
|
|
proc->proc_ompi = 0;
|
|
proc->proc_addrs = NULL;
|
|
proc->proc_addr_count = 0;
|
|
proc->proc_endpoints = NULL;
|
|
proc->proc_endpoint_count = 0;
|
|
OBJ_CONSTRUCT(&proc->proc_lock, opal_mutex_t);
|
|
}
|
|
|
|
/*
|
|
* Cleanup ib proc instance
|
|
*/
|
|
|
|
void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t* proc)
|
|
{
|
|
/* remove from list of all proc instances */
|
|
OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
|
|
orte_hash_table_remove_proc(&mca_btl_tcp_component.tcp_procs, &proc->proc_name);
|
|
OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
|
|
|
|
/* release resources */
|
|
if(NULL != proc->proc_endpoints) {
|
|
free(proc->proc_endpoints);
|
|
OBJ_DESTRUCT(&proc->proc_lock);
|
|
}
|
|
}
|
|
|
|
|
|
/*
|
|
* Check to see if an IPv4 struct in_addr is public or private. We
|
|
* can only do IPv4 here because some of the TCP BTL endpoint structs
|
|
* only hold the struct in_addr, not the upper-level sin_family that
|
|
* would indicate if the address is IPv6.
|
|
*/
|
|
static bool is_private_ipv4(struct in_addr *in)
|
|
{
|
|
/* There are definitely ways to do this more efficiently, but
|
|
since this is not performance-critical code, it seems better to
|
|
use clear code (vs. clever code) */
|
|
|
|
uint32_t addr = ntohl((uint32_t) in->s_addr);
|
|
unsigned int a = (addr & 0xff000000) >> 24;
|
|
unsigned int b = (addr & 0x00ff0000) >> 16;
|
|
|
|
return ((10 == a) ||
|
|
(192 == a && 168 == b) ||
|
|
(172 == a && 16 == b)) ? true : false;
|
|
}
|
|
|
|
|
|
/*
|
|
* Create a TCP process structure. There is a one-to-one correspondence
|
|
* between a ompi_proc_t and a mca_btl_tcp_proc_t instance. We cache
|
|
* additional data (specifically the list of mca_btl_tcp_endpoint_t instances,
|
|
* and published addresses) associated w/ a given destination on this
|
|
* datastructure.
|
|
*/
|
|
|
|
mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(ompi_proc_t* ompi_proc)
|
|
{
|
|
int rc;
|
|
size_t size;
|
|
mca_btl_tcp_proc_t* btl_proc;
|
|
|
|
OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
|
|
btl_proc = (mca_btl_tcp_proc_t*)orte_hash_table_get_proc(
|
|
&mca_btl_tcp_component.tcp_procs, &ompi_proc->proc_name);
|
|
if(NULL != btl_proc) {
|
|
OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
|
|
return btl_proc;
|
|
}
|
|
|
|
btl_proc = OBJ_NEW(mca_btl_tcp_proc_t);
|
|
if(NULL == btl_proc)
|
|
return NULL;
|
|
btl_proc->proc_ompi = ompi_proc;
|
|
btl_proc->proc_name = ompi_proc->proc_name;
|
|
|
|
/* add to hash table of all proc instance */
|
|
orte_hash_table_set_proc(
|
|
&mca_btl_tcp_component.tcp_procs,
|
|
&btl_proc->proc_name,
|
|
btl_proc);
|
|
OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
|
|
|
|
/* lookup tcp parameters exported by this proc */
|
|
rc = mca_pml_base_modex_recv( &mca_btl_tcp_component.super.btl_version,
|
|
ompi_proc,
|
|
(void**)&btl_proc->proc_addrs,
|
|
&size );
|
|
if(rc != OMPI_SUCCESS) {
|
|
BTL_ERROR(("mca_base_modex_recv: failed with return value=%d", rc));
|
|
OBJ_RELEASE(btl_proc);
|
|
return NULL;
|
|
}
|
|
if(0 != (size % sizeof(mca_btl_tcp_addr_t))) {
|
|
BTL_ERROR(("mca_base_modex_recv: invalid size %d\n", size));
|
|
return NULL;
|
|
}
|
|
btl_proc->proc_addr_count = size / sizeof(mca_btl_tcp_addr_t);
|
|
|
|
/* allocate space for endpoint array - one for each exported address */
|
|
btl_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
|
|
malloc(btl_proc->proc_addr_count * sizeof(mca_btl_base_endpoint_t*));
|
|
if(NULL == btl_proc->proc_endpoints) {
|
|
OBJ_RELEASE(btl_proc);
|
|
return NULL;
|
|
}
|
|
if(NULL == mca_btl_tcp_component.tcp_local && ompi_proc == ompi_proc_local())
|
|
mca_btl_tcp_component.tcp_local = btl_proc;
|
|
return btl_proc;
|
|
}
|
|
|
|
|
|
/*
|
|
* Note that this routine must be called with the lock on the process
|
|
* already held. Insert a btl instance into the proc array and assign
|
|
* it an address.
|
|
*/
|
|
int mca_btl_tcp_proc_insert(
|
|
mca_btl_tcp_proc_t* btl_proc,
|
|
mca_btl_base_endpoint_t* btl_endpoint)
|
|
{
|
|
struct mca_btl_tcp_module_t *btl_tcp = btl_endpoint->endpoint_btl;
|
|
size_t i;
|
|
unsigned long net1;
|
|
|
|
/* insert into endpoint array */
|
|
btl_endpoint->endpoint_proc = btl_proc;
|
|
btl_proc->proc_endpoints[btl_proc->proc_endpoint_count++] = btl_endpoint;
|
|
|
|
net1 = btl_tcp->tcp_ifaddr.sin_addr.s_addr & btl_tcp->tcp_ifmask.sin_addr.s_addr;
|
|
|
|
/*
|
|
* Look through the proc instance for an address that is on the
|
|
* directly attached network. If we don't find one, pick the first
|
|
* unused address.
|
|
*/
|
|
for(i=0; i<btl_proc->proc_addr_count; i++) {
|
|
mca_btl_tcp_addr_t* endpoint_addr = btl_proc->proc_addrs + i;
|
|
unsigned long net2 = endpoint_addr->addr_inet.s_addr & btl_tcp->tcp_ifmask.sin_addr.s_addr;
|
|
if(endpoint_addr->addr_inuse != 0)
|
|
continue;
|
|
if(net1 == net2) {
|
|
btl_endpoint->endpoint_addr = endpoint_addr;
|
|
break;
|
|
} else if(btl_endpoint->endpoint_addr != 0) {
|
|
btl_endpoint->endpoint_addr = endpoint_addr;
|
|
}
|
|
}
|
|
|
|
/* Make sure there is a common interface */
|
|
if( NULL != btl_endpoint->endpoint_addr ) {
|
|
btl_endpoint->endpoint_addr->addr_inuse++;
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
/* There was no common interface. So what do we do? For the
|
|
moment, we'll do enough to cover 2 common cases:
|
|
|
|
1. Running MPI processes on two computers that are not on the
|
|
same subnet, but still have routable addresses to each
|
|
other. In this case, the above subnet matching will fail,
|
|
but since the addresses are routable, the
|
|
OS/networking/routers will make it all work ok. So we need
|
|
to make this function *not* return OMPI_ERR_UNREACH.
|
|
|
|
2. Running MPI processes on a typical cluster configuration
|
|
where a head node has 2 TCP NICs (one public IP address one
|
|
private IP address) and all the back-end compute nodes have
|
|
only private IP addresses. In this scenario, the MPI
|
|
process on the head node will have 2 TCP BTL modules (one
|
|
for the public, one for the private). The module with the
|
|
private IP address will match the subnet and all will work
|
|
fine. The module with the public IP address will not match
|
|
anything and fall through to here -- we want it to return
|
|
OMPI_ERR_UNREACH so that that module will effectively have
|
|
no peers that it can communicate with.
|
|
|
|
To support these two scenarios, do the following:
|
|
|
|
- if my address is private (10., 192.168., or 172.16.), return
|
|
UNREACH.
|
|
- if my address is public, return the first public address from
|
|
my peer (and hope for the best), or UNREACH if there are none
|
|
available.
|
|
|
|
This does not cover some other scenarios that we'll likely need
|
|
to support in the future, such as:
|
|
|
|
- Flat neighborhood networks -- where all the IP's in question
|
|
are private, the subnet masking won't necessarily match, but
|
|
they're routable to each other.
|
|
- Really large, private TCP-based clusters, such as a 1024 node
|
|
TCP-based cluster. Depending on how the subnet masks are set
|
|
by the admins, there may be a subnet mask that effectively
|
|
spans the entire cluster, or (for example) subnet masks may
|
|
be set such that only nodes on the same switches are on the
|
|
same subnet. This latter scenario will not be supported
|
|
by the above cases.
|
|
|
|
To support these kinds of scenarios, we really need "something
|
|
better", such as allowing the user to specify a config file
|
|
indicating which subnets are reachable by which interface, etc.
|
|
*/
|
|
|
|
else {
|
|
/* If my address is private, return UNREACH */
|
|
if (is_private_ipv4(&(btl_tcp->tcp_ifaddr.sin_addr))) {
|
|
return OMPI_ERR_UNREACH;
|
|
}
|
|
|
|
/* Find the first public peer address */
|
|
for (i = 0; i < btl_proc->proc_addr_count; ++i) {
|
|
mca_btl_tcp_addr_t* endpoint_addr = btl_proc->proc_addrs + i;
|
|
if (!is_private_ipv4(&(endpoint_addr->addr_inet))) {
|
|
btl_endpoint->endpoint_addr = endpoint_addr;
|
|
btl_endpoint->endpoint_addr->addr_inuse++;
|
|
return OMPI_SUCCESS;
|
|
}
|
|
}
|
|
|
|
/* Didn't find any peer addresses that were public, so return
|
|
UNREACH */
|
|
return OMPI_ERR_UNREACH;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Remove an endpoint from the proc array and indicate the address is
|
|
* no longer in use.
|
|
*/
|
|
|
|
int mca_btl_tcp_proc_remove(mca_btl_tcp_proc_t* btl_proc, mca_btl_base_endpoint_t* btl_endpoint)
|
|
{
|
|
size_t i;
|
|
OPAL_THREAD_LOCK(&btl_proc->proc_lock);
|
|
for(i=0; i<btl_proc->proc_endpoint_count; i++) {
|
|
if(btl_proc->proc_endpoints[i] == btl_endpoint) {
|
|
memmove(btl_proc->proc_endpoints+i, btl_proc->proc_endpoints+i+1,
|
|
(btl_proc->proc_endpoint_count-i-1)*sizeof(mca_btl_base_endpoint_t*));
|
|
if(--btl_proc->proc_endpoint_count == 0) {
|
|
OPAL_THREAD_UNLOCK(&btl_proc->proc_lock);
|
|
OBJ_RELEASE(btl_proc);
|
|
return OMPI_SUCCESS;
|
|
}
|
|
btl_endpoint->endpoint_addr->addr_inuse--;
|
|
break;
|
|
}
|
|
}
|
|
OPAL_THREAD_UNLOCK(&btl_proc->proc_lock);
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
/*
|
|
* Look for an existing TCP process instance based on the globally unique
|
|
* process identifier.
|
|
*/
|
|
mca_btl_tcp_proc_t* mca_btl_tcp_proc_lookup(const orte_process_name_t *name)
|
|
{
|
|
mca_btl_tcp_proc_t* proc;
|
|
OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
|
|
proc = (mca_btl_tcp_proc_t*)orte_hash_table_get_proc(
|
|
&mca_btl_tcp_component.tcp_procs, name);
|
|
OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
|
|
return proc;
|
|
}
|
|
|
|
/*
|
|
* loop through all available PTLs for one matching the source address
|
|
* of the request.
|
|
*/
|
|
bool mca_btl_tcp_proc_accept(mca_btl_tcp_proc_t* btl_proc, struct sockaddr_in* addr, int sd)
|
|
{
|
|
size_t i;
|
|
OPAL_THREAD_LOCK(&btl_proc->proc_lock);
|
|
for(i=0; i<btl_proc->proc_endpoint_count; i++) {
|
|
mca_btl_base_endpoint_t* btl_endpoint = btl_proc->proc_endpoints[i];
|
|
if(mca_btl_tcp_endpoint_accept(btl_endpoint, addr, sd)) {
|
|
OPAL_THREAD_UNLOCK(&btl_proc->proc_lock);
|
|
return true;
|
|
}
|
|
}
|
|
OPAL_THREAD_UNLOCK(&btl_proc->proc_lock);
|
|
return false;
|
|
}
|
|
|
|
|