/*
* Copyright (c) 2015-2016 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "orte_config.h"
#include "orte/constants.h"
#include "opal/mca/base/base.h"
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "opal/mca/backtrace/backtrace.h"
#include "opal/mca/event/event.h"
#if OPAL_ENABLE_FT_CR == 1
#include "orte/mca/rml/rml.h"
#include "orte/mca/state/state.h"
#endif
#include "orte/mca/rml/base/base.h"
#include "orte/mca/rml/rml_types.h"
#include "orte/mca/routed/routed.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/util/name_fns.h"
#include "orte/runtime/orte_globals.h"
#include "rml_ofi.h"
static int rml_ofi_component_open(void);
static int rml_ofi_component_close(void);
static int rml_ofi_component_init(void);
static orte_rml_base_module_t* open_conduit(opal_list_t *attributes);
static orte_rml_pathway_t* query_transports(void);
static char* ofi_get_contact_info(void);
static void process_uri(char *uri);
static void ofi_set_contact_info (const char *uri);
void convert_to_sockaddr( char *ofiuri, struct sockaddr_in* ep_sockaddr);
/**
* component definition
*/
orte_rml_component_t mca_rml_ofi_component = {
/* First, the mca_base_component_t struct containing meta
information about the component itself */
.base = {
ORTE_RML_BASE_VERSION_3_0_0,
.mca_component_name = "ofi",
MCA_BASE_MAKE_VERSION(component, ORTE_MAJOR_VERSION, ORTE_MINOR_VERSION,
ORTE_RELEASE_VERSION),
.mca_open_component = rml_ofi_component_open,
.mca_close_component = rml_ofi_component_close,
},
.data = {
/* The component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
},
.priority = 10,
.open_conduit = open_conduit,
.query_transports = query_transports,
.get_contact_info = ofi_get_contact_info,
.set_contact_info = ofi_set_contact_info,
.close_conduit = NULL
};
/* Local variables */
orte_rml_ofi_module_t orte_rml_ofi = {
.api = {
.component = (struct orte_rml_component_t*)&mca_rml_ofi_component,
.ping = NULL,
.send_nb = orte_rml_ofi_send_nb,
.send_buffer_nb = orte_rml_ofi_send_buffer_nb,
.purge = NULL
}
};
/* Local variables */
static bool init_done = false;
static int
rml_ofi_component_open(void)
{
/* Initialise endpoint and all queues */
orte_rml_ofi.fi_info_list = NULL;
orte_rml_ofi.min_ofi_recv_buf_sz = MIN_MULTI_BUF_SIZE;
orte_rml_ofi.cur_msgid = 1;
orte_rml_ofi.cur_transport_id = RML_OFI_PROV_ID_INVALID;
orte_rml_ofi.ofi_prov_open_num = 0;
OBJ_CONSTRUCT(&orte_rml_ofi.peers, opal_hash_table_t);
opal_hash_table_init(&orte_rml_ofi.peers, 128);
for( uint8_t ofi_prov_id=0; ofi_prov_id < MAX_OFI_PROVIDERS ; ofi_prov_id++) {
orte_rml_ofi.ofi_prov[ofi_prov_id].fabric = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].domain = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].av = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].cq = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].ep = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].ep_name[0] = 0;
orte_rml_ofi.ofi_prov[ofi_prov_id].epnamelen = 0;
orte_rml_ofi.ofi_prov[ofi_prov_id].mr_multi_recv = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].rxbuf = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].rxbuf_size = 0;
orte_rml_ofi.ofi_prov[ofi_prov_id].progress_ev_active = false;
orte_rml_ofi.ofi_prov[ofi_prov_id].ofi_prov_id = RML_OFI_PROV_ID_INVALID;
}
opal_output_verbose(10,orte_rml_base_framework.framework_output," from %s:%d rml_ofi_component_open()",__FILE__,__LINE__);
return ORTE_SUCCESS;
}
void free_ofi_prov_resources( int ofi_prov_id)
{
int ret=0;
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - free_ofi_prov_resources() begin. OFI ofi_prov_id- %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),ofi_prov_id);
if (orte_rml_ofi.ofi_prov[ofi_prov_id].ep) {
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - close ep",ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
CLOSE_FID(orte_rml_ofi.ofi_prov[ofi_prov_id].ep);
if (ret)
{
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - fi_close(ep) failed with error- %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),ret);
}
}
if (orte_rml_ofi.ofi_prov[ofi_prov_id].mr_multi_recv) {
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - close mr_multi_recv",ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
CLOSE_FID(orte_rml_ofi.ofi_prov[ofi_prov_id].mr_multi_recv);
}
if (orte_rml_ofi.ofi_prov[ofi_prov_id].cq) {
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - close cq",ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
CLOSE_FID(orte_rml_ofi.ofi_prov[ofi_prov_id].cq);
}
if (orte_rml_ofi.ofi_prov[ofi_prov_id].av) {
CLOSE_FID(orte_rml_ofi.ofi_prov[ofi_prov_id].av);
}
if (orte_rml_ofi.ofi_prov[ofi_prov_id].domain) {
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - close domain",ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
CLOSE_FID(orte_rml_ofi.ofi_prov[ofi_prov_id].domain);
}
if (orte_rml_ofi.ofi_prov[ofi_prov_id].fabric) {
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - close fabric",ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
fi_close((fid_t)orte_rml_ofi.ofi_prov[ofi_prov_id].fabric);
}
if (orte_rml_ofi.ofi_prov[ofi_prov_id].rxbuf) {
free(orte_rml_ofi.ofi_prov[ofi_prov_id].rxbuf);
}
orte_rml_ofi.ofi_prov[ofi_prov_id].fabric = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].domain = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].av = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].cq = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].ep = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].ep_name[0] = 0;
orte_rml_ofi.ofi_prov[ofi_prov_id].epnamelen = 0;
orte_rml_ofi.ofi_prov[ofi_prov_id].rxbuf = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].rxbuf_size = 0;
orte_rml_ofi.ofi_prov[ofi_prov_id].fabric_info = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].mr_multi_recv = NULL;
orte_rml_ofi.ofi_prov[ofi_prov_id].ofi_prov_id = RML_OFI_PROV_ID_INVALID;
OPAL_LIST_DESTRUCT(&orte_rml_ofi.recv_msg_queue_list);
if( orte_rml_ofi.ofi_prov[ofi_prov_id].progress_ev_active) {
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - deleting progress event",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
opal_event_del( &orte_rml_ofi.ofi_prov[ofi_prov_id].progress_event);
}
return;
}
static int
rml_ofi_component_close(void)
{
int rc;
opal_object_t *value;
uint64_t key;
void *node;
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - rml_ofi_component_close() -begin, total open OFI providers = %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),orte_rml_ofi.ofi_prov_open_num);
if(orte_rml_ofi.fi_info_list) {
(void) fi_freeinfo(orte_rml_ofi.fi_info_list);
}
/* Close endpoint and all queues */
for( uint8_t ofi_prov_id=0;ofi_prov_id<orte_rml_ofi.ofi_prov_open_num;ofi_prov_id++) {
free_ofi_prov_resources(ofi_prov_id);
}
/* release all peers from the hash table */
rc = opal_hash_table_get_first_key_uint64 (&orte_rml_ofi.peers, &key,
(void **) &value, &node);
while (OPAL_SUCCESS == rc) {
if (NULL != value) {
OBJ_RELEASE(value);
}
rc = opal_hash_table_get_next_key_uint64 (&orte_rml_ofi.peers, &key,
(void **) &value, node, &node);
}
/* release all peers from the hash table */
rc = opal_hash_table_get_first_key_uint64 (&orte_rml_ofi.peers, &key,
(void **) &value, &node);
while (OPAL_SUCCESS == rc) {
if (NULL != value) {
OBJ_RELEASE(value);
}
rc = opal_hash_table_get_next_key_uint64 (&orte_rml_ofi.peers, &key,
(void **) &value, node, &node);
}
OBJ_DESTRUCT(&orte_rml_ofi.peers);
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - rml_ofi_component_close() end",ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return ORTE_SUCCESS;
}
void print_provider_info (struct fi_info *cur_fi )
{
//Display all the details in the fi_info structure
opal_output_verbose(1,orte_rml_base_framework.framework_output,
" %s - Print_provider_info() ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" Provider name : %s",cur_fi->fabric_attr->prov_name);
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" Protocol : %s",fi_tostr(&cur_fi->ep_attr->protocol,FI_TYPE_PROTOCOL));
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" EP Type : %s",fi_tostr(&cur_fi->ep_attr->type,FI_TYPE_EP_TYPE));
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" address_format : %s",fi_tostr(&cur_fi->addr_format,FI_TYPE_ADDR_FORMAT));
}
void print_provider_list_info (struct fi_info *fi )
{
struct fi_info *cur_fi = fi;
int fi_count = 0;
//Display all the details in the fi_info structure
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %s - Print_provider_list_info() ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
while( NULL != cur_fi ) {
fi_count++;
opal_output_verbose(10,orte_rml_base_framework.framework_output,
" %d.\n",fi_count);
print_provider_info( cur_fi);
cur_fi = cur_fi->next;
}
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"Total # of providers supported is %d\n",fi_count);
}
/*
* This returns all the supported transports in the system that support endpoint type RDM (reliable datagram)
* The providers returned is a list of type opal_valut_t holding opal_list_t
*/
static orte_rml_pathway_t* query_transports(void)
{
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"%s:%d OFI Query Interface not implemented",__FILE__,__LINE__);
return NULL;
}
/**
ofi_prov [in]: the ofi ofi_prov_id that triggered the progress fn
**/
__opal_attribute_always_inline__ static inline int
orte_rml_ofi_progress(ofi_transport_ofi_prov_t* prov)
{
ssize_t ret;
int count=0; /* number of messages read and processed */
struct fi_cq_data_entry wc = { 0 };
struct fi_cq_err_entry error = { 0 };
orte_rml_ofi_request_t *ofi_req;
opal_output_verbose(10, orte_rml_base_framework.framework_output,
"%s orte_rml_ofi_progress called for OFI ofi_provid %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id);
/**
* Read the work completions from the CQ.
* From the completion's op_context, we get the associated OFI request.
* Call the request's callback.
*/
while (true) {
/* Read the cq - that triggered the libevent to call this progress fn. */
ret = fi_cq_read(prov->cq, (void *)&wc, 1);
if (0 < ret) {
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s cq read for OFI ofi_provid %d - wc.flags = %llx",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id, (long long unsigned int)wc.flags);
count++;
// check the flags to see if this is a send-completion or receive
if ( wc.flags & FI_SEND )
{
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s Send completion received on OFI provider id %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id);
if (NULL != wc.op_context) {
/* get the context from the wc and call the message handler */
ofi_req = TO_OFI_REQ(wc.op_context);
assert(ofi_req);
ret = orte_rml_ofi_send_callback(&wc, ofi_req);
if (ORTE_SUCCESS != ret) {
opal_output(orte_rml_base_framework.framework_output,
"Error returned by OFI send callback handler when a send completion was received on OFI prov: %zd",
ret);
}
}
} else if ( (wc.flags & FI_RECV) && (wc.flags & FI_MULTI_RECV) ) {
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s Received message on OFI ofi_prov_id %d - but buffer is consumed, need to repost",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id);
// reposting buffer
ret = fi_recv(orte_rml_ofi.ofi_prov[prov->ofi_prov_id].ep,
orte_rml_ofi.ofi_prov[prov->ofi_prov_id].rxbuf,
orte_rml_ofi.ofi_prov[prov->ofi_prov_id].rxbuf_size,
fi_mr_desc(orte_rml_ofi.ofi_prov[prov->ofi_prov_id].mr_multi_recv),
0,&(prov->rx_ctx1));
// call the receive message handler that will call the rml_base
ret = orte_rml_ofi_recv_handler(&wc, prov->ofi_prov_id);
if (ORTE_SUCCESS != ret) {
opal_output(orte_rml_base_framework.framework_output,
"Error returned by OFI Recv handler when handling the received message on the prov: %zd",
ret);
}
} else if ( wc.flags & FI_RECV ) {
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s Received message on OFI provider id %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id);
// call the receive message handler that will call the rml_base
ret = orte_rml_ofi_recv_handler(&wc, prov->ofi_prov_id);
if (ORTE_SUCCESS != ret) {
opal_output(orte_rml_base_framework.framework_output,
"Error returned by OFI Recv handler when handling the received message on the OFI prov: %zd",
ret);
}
} else if ( wc.flags & FI_MULTI_RECV ) {
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s Received buffer overrun message on OFI provider id %d - need to repost",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id);
// reposting buffer
ret = fi_recv(orte_rml_ofi.ofi_prov[prov->ofi_prov_id].ep,
orte_rml_ofi.ofi_prov[prov->ofi_prov_id].rxbuf,
orte_rml_ofi.ofi_prov[prov->ofi_prov_id].rxbuf_size,
fi_mr_desc(orte_rml_ofi.ofi_prov[prov->ofi_prov_id].mr_multi_recv),
0,&(prov->rx_ctx1));
if (ORTE_SUCCESS != ret) {
opal_output(orte_rml_base_framework.framework_output,
"Error returned by OFI when reposting buffer on the OFI prov: %zd",
ret);
}
}else {
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"CQ has unhandled completion event with FLAG wc.flags = 0x%llx",
(long long unsigned int)wc.flags);
}
} else if (ret == -FI_EAVAIL) {
/**
* An error occured and is being reported via the CQ.
* Read the error and forward it to the upper layer.
*/
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s cq_read for OFI provider id %d returned error 0x%zx <%s>",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id, ret,
fi_strerror((int) -ret) );
ret = fi_cq_readerr(prov->cq,&error,0);
if (0 > ret) {
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"Error returned from fi_cq_readerr: %zd", ret);
}
assert(error.op_context);
/* get the context from wc and call the error handler */
ofi_req = TO_OFI_REQ(error.op_context);
assert(ofi_req);
ret = orte_rml_ofi_error_callback(&error, ofi_req);
if (ORTE_SUCCESS != ret) {
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"Error returned by request error callback: %zd",
ret);
}
break;
} else if (ret == -FI_EAGAIN){
/**
* The CQ is empty. Return.
*/
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s Empty cq for OFI provider id %d,exiting from ofi_progress()",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id );
break;
} else {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s cq_read for OFI provider id %d returned error 0x%zx <%s>",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id, ret,
fi_strerror((int) -ret) );
break;
}
}
return count;
}
/*
* call the ofi_progress() fn to read the cq
*
*/
int cq_progress_handler(int sd, short flags, void *cbdata)
{
ofi_transport_ofi_prov_t* prov = (ofi_transport_ofi_prov_t*)cbdata;
int count;
opal_output_verbose(10, orte_rml_base_framework.framework_output,
"%s cq_progress_handler called for OFI Provider id %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
prov->ofi_prov_id);
/* call the progress fn to read the cq and process the message
* for the ofi provider */
count = orte_rml_ofi_progress(prov);
return count;
}
/*
* Returns the number of ofi-providers available
*/
static int rml_ofi_component_init(void)
{
int ret, fi_version;
struct fi_info *hints, *fabric_info;
struct fi_cq_attr cq_attr = {0};
struct fi_av_attr av_attr = {0};
char *pmix_key;
uint8_t cur_ofi_prov;
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"%s - Entering rml_ofi_component_init()",ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
if (init_done) {
return orte_rml_ofi.ofi_prov_open_num;
}
/**
* Hints to filter providers
* See man fi_getinfo for a list of all filters
* mode: Select capabilities MTL is prepared to support.
* In this case, MTL will pass in context into communication calls
* ep_type: reliable datagram operation
* caps: Capabilities required from the provider.
* Tag matching is specified to implement MPI semantics.
* msg_order: Guarantee that messages with same tag are ordered.
*/
hints = fi_allocinfo();
if (!hints) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: Could not allocate fi_info\n",
__FILE__, __LINE__);
return orte_rml_ofi.ofi_prov_open_num;
}
/**
* Refine filter for additional capabilities
* endpoint type : Reliable datagram
* threading: Disable locking
* control_progress: enable async progress
*/
hints->mode = FI_CONTEXT;
hints->ep_attr->type = FI_EP_RDM; /* Reliable datagram */
hints->domain_attr->threading = FI_THREAD_UNSPEC;
hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
hints->domain_attr->av_type = FI_AV_MAP;
/**
* FI_VERSION provides binary backward and forward compatibility support
* Specify the version of OFI is coded to, the provider will select struct
* layouts that are compatible with this version.
*/
fi_version = FI_VERSION(1, 3);
/**
* fi_getinfo: returns information about fabric services for reaching a
* remote node or service. this does not necessarily allocate resources.
* Pass NULL for name/service because we want a list of providers supported.
*/
ret = fi_getinfo(fi_version, /* OFI version requested */
NULL, /* Optional name or fabric to resolve */
NULL, /* Optional service name or port to request */
0ULL, /* Optional flag */
hints, /* In: Hints to filter providers */
&orte_rml_ofi.fi_info_list); /* Out: List of matching providers */
if (0 != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_getinfo failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
} else {
/* added for debug purpose - Print the provider info
print_transports_query();
print_provider_list_info(orte_rml_ofi.fi_info_list);
*/
/** create the OFI objects for each transport in the system
* (fi_info_list) and store it in the ofi_prov array **/
orte_rml_ofi.ofi_prov_open_num = 0; // start the ofi_prov_id from 0
for( fabric_info = orte_rml_ofi.fi_info_list ;
NULL != fabric_info && orte_rml_ofi.ofi_prov_open_num < MAX_OFI_PROVIDERS ; fabric_info = fabric_info->next)
{
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"%s:%d beginning to add endpoint for OFI_provider_id=%d ",__FILE__,__LINE__,orte_rml_ofi.ofi_prov_open_num);
print_provider_info(fabric_info);
cur_ofi_prov = orte_rml_ofi.ofi_prov_open_num;
orte_rml_ofi.ofi_prov[cur_ofi_prov].ofi_prov_id = orte_rml_ofi.ofi_prov_open_num ;
orte_rml_ofi.ofi_prov[cur_ofi_prov].fabric_info = fabric_info;
// set FI_MULTI_RECV flag for all recv operations
fabric_info->rx_attr->op_flags = FI_MULTI_RECV;
/**
* Open fabric
* The getinfo struct returns a fabric attribute struct that can be used to
* instantiate the virtual or physical network. This opens a "fabric
* provider". See man fi_fabric for details.
*/
ret = fi_fabric(fabric_info->fabric_attr, /* In: Fabric attributes */
&orte_rml_ofi.ofi_prov[cur_ofi_prov].fabric, /* Out: Fabric handle */
NULL); /* Optional context for fabric events */
if (0 != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_fabric failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
orte_rml_ofi.ofi_prov[cur_ofi_prov].fabric = NULL;
/* abort this current transport, but check if next transport can be opened */
continue;
}
/**
* Create the access domain, which is the physical or virtual network or
* hardware port/collection of ports. Returns a domain object that can be
* used to create endpoints. See man fi_domain for details.
*/
ret = fi_domain(orte_rml_ofi.ofi_prov[cur_ofi_prov].fabric, /* In: Fabric object */
fabric_info, /* In: Provider */
&orte_rml_ofi.ofi_prov[cur_ofi_prov].domain, /* Out: Domain oject */
NULL); /* Optional context for domain events */
if (0 != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_domain failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
orte_rml_ofi.ofi_prov[cur_ofi_prov].domain = NULL;
/* abort this current transport, but check if next transport can be opened */
continue;
}
/**
* Create a transport level communication endpoint. To use the endpoint,
* it must be bound to completion counters or event queues and enabled,
* and the resources consumed by it, such as address vectors, counters,
* completion queues, etc.
* see man fi_endpoint for more details.
*/
ret = fi_endpoint(orte_rml_ofi.ofi_prov[cur_ofi_prov].domain, /* In: Domain object */
fabric_info, /* In: Provider */
&orte_rml_ofi.ofi_prov[cur_ofi_prov].ep, /* Out: Endpoint object */
NULL); /* Optional context */
if (0 != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_endpoint failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
/**
* Save the maximum inject size.
*/
//orte_rml_ofi.max_inject_size = prov->tx_attr->inject_size;
/**
* Create the objects that will be bound to the endpoint.
* The objects include:
* - completion queue for events
* - address vector of other endpoint addresses
* - dynamic memory-spanning memory region
*/
cq_attr.format = FI_CQ_FORMAT_DATA;
cq_attr.wait_obj = FI_WAIT_FD;
cq_attr.wait_cond = FI_CQ_COND_NONE;
ret = fi_cq_open(orte_rml_ofi.ofi_prov[cur_ofi_prov].domain,
&cq_attr, &orte_rml_ofi.ofi_prov[cur_ofi_prov].cq, NULL);
if (ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_cq_open failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
/**
* The remote fi_addr will be stored in the ofi_endpoint struct.
* So, we use the AV in "map" mode.
*/
av_attr.type = FI_AV_MAP;
ret = fi_av_open(orte_rml_ofi.ofi_prov[cur_ofi_prov].domain,
&av_attr, &orte_rml_ofi.ofi_prov[cur_ofi_prov].av, NULL);
if (ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_av_open failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
/**
* Bind the CQ and AV to the endpoint object.
*/
ret = fi_ep_bind(orte_rml_ofi.ofi_prov[cur_ofi_prov].ep,
(fid_t)orte_rml_ofi.ofi_prov[cur_ofi_prov].cq,
FI_SEND | FI_RECV);
if (0 != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_bind CQ-EP failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
ret = fi_ep_bind(orte_rml_ofi.ofi_prov[cur_ofi_prov].ep,
(fid_t)orte_rml_ofi.ofi_prov[cur_ofi_prov].av,
0);
if (0 != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_bind AV-EP failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
/**
* Enable the endpoint for communication
* This commits the bind operations.
*/
ret = fi_enable(orte_rml_ofi.ofi_prov[cur_ofi_prov].ep);
if (0 != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_enable failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"%s:%d ep enabled for ofi_prov_id - %d ",__FILE__,__LINE__,orte_rml_ofi.ofi_prov[cur_ofi_prov].ofi_prov_id);
/**
* Get our address and publish it with modex.
**/
orte_rml_ofi.ofi_prov[cur_ofi_prov].epnamelen = sizeof (orte_rml_ofi.ofi_prov[cur_ofi_prov].ep_name);
ret = fi_getname((fid_t)orte_rml_ofi.ofi_prov[cur_ofi_prov].ep,
&orte_rml_ofi.ofi_prov[cur_ofi_prov].ep_name[0],
&orte_rml_ofi.ofi_prov[cur_ofi_prov].epnamelen);
if (ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_getname failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
/* Register the ofi address of this peer with PMIX server only if it is a user process /
* for daemons the set/get_contact_info is used to exchange this information */
if (ORTE_PROC_IS_APP) {
asprintf(&pmix_key,"%s%d",orte_rml_ofi.ofi_prov[cur_ofi_prov].fabric_info->fabric_attr->prov_name,cur_ofi_prov);
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s calling OPAL_MODEX_SEND_STRING for key - %s ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), pmix_key );
OPAL_MODEX_SEND_STRING( ret, OPAL_PMIX_GLOBAL,
pmix_key,
orte_rml_ofi.ofi_prov[cur_ofi_prov].ep_name,
orte_rml_ofi.ofi_prov[cur_ofi_prov].epnamelen);
/*print debug information on opal_modex_string */
switch ( orte_rml_ofi.ofi_prov[cur_ofi_prov].fabric_info->addr_format)
{
case FI_SOCKADDR_IN :
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"%s:%d In FI_SOCKADDR_IN. ",__FILE__,__LINE__);
/* Address is of type sockaddr_in (IPv4) */
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"%s sending Opal modex string for ofi prov_id %d, epnamelen = %lu ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),cur_ofi_prov,orte_rml_ofi.ofi_prov[cur_ofi_prov].epnamelen);
/*[debug] - print the sockaddr - port and s_addr */
struct sockaddr_in* ep_sockaddr = (struct sockaddr_in*)orte_rml_ofi.ofi_prov[cur_ofi_prov].ep_name;
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"%s port = 0x%x, InternetAddr = 0x%s ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),ntohs(ep_sockaddr->sin_port),inet_ntoa(ep_sockaddr->sin_addr));
break;
}
/* end of printing opal_modex_string and port, IP */
free(pmix_key);
if (ORTE_SUCCESS != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: OPAL_MODEX_SEND failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/*abort this current transport, but check if next transport can be opened*/
continue;
}
}
/**
* Set the ANY_SRC address.
*/
orte_rml_ofi.any_addr = FI_ADDR_UNSPEC;
/**
* Allocate tx,rx buffers and Post a multi-RECV buffer for each endpoint
**/
//[TODO later] For now not considering ep_attr prefix_size (add this later)
orte_rml_ofi.ofi_prov[cur_ofi_prov].rxbuf_size = MIN_MULTI_BUF_SIZE * MULTI_BUF_SIZE_FACTOR;
orte_rml_ofi.ofi_prov[cur_ofi_prov].rxbuf = malloc(orte_rml_ofi.ofi_prov[cur_ofi_prov].rxbuf_size);
ret = fi_mr_reg(orte_rml_ofi.ofi_prov[cur_ofi_prov].domain,
orte_rml_ofi.ofi_prov[cur_ofi_prov].rxbuf,
orte_rml_ofi.ofi_prov[cur_ofi_prov].rxbuf_size,
FI_RECV, 0, 0, 0, &orte_rml_ofi.ofi_prov[cur_ofi_prov].mr_multi_recv,
&orte_rml_ofi.ofi_prov[cur_ofi_prov].rx_ctx1);
if (ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_mr_reg failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
ret = fi_setopt(&orte_rml_ofi.ofi_prov[cur_ofi_prov].ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
&orte_rml_ofi.min_ofi_recv_buf_sz, sizeof(orte_rml_ofi.min_ofi_recv_buf_sz) );
if (ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_setopt failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
ret = fi_recv(orte_rml_ofi.ofi_prov[cur_ofi_prov].ep,
orte_rml_ofi.ofi_prov[cur_ofi_prov].rxbuf,
orte_rml_ofi.ofi_prov[cur_ofi_prov].rxbuf_size,
fi_mr_desc(orte_rml_ofi.ofi_prov[cur_ofi_prov].mr_multi_recv),
0,&orte_rml_ofi.ofi_prov[cur_ofi_prov].rx_ctx1);
if (ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_recv failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
/**
* get the fd and register the progress fn
**/
ret = fi_control(&orte_rml_ofi.ofi_prov[cur_ofi_prov].cq->fid, FI_GETWAIT,
(void *) &orte_rml_ofi.ofi_prov[cur_ofi_prov].fd);
if (0 != ret) {
opal_output_verbose(1, orte_rml_base_framework.framework_output,
"%s:%d: fi_control failed to get fd: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
free_ofi_prov_resources(cur_ofi_prov);
/* abort this current transport, but check if next transport can be opened */
continue;
}
/* - create the event that will wait on the fd*/
/* use the opal_event_set to do a libevent set on the fd
* so when something is available to read, the cq_porgress_handler
* will be called */
opal_event_set(orte_event_base,
&orte_rml_ofi.ofi_prov[cur_ofi_prov].progress_event,
orte_rml_ofi.ofi_prov[cur_ofi_prov].fd,
OPAL_EV_READ|OPAL_EV_PERSIST,
cq_progress_handler,
&orte_rml_ofi.ofi_prov[cur_ofi_prov]);
opal_event_add(&orte_rml_ofi.ofi_prov[cur_ofi_prov].progress_event, 0);
orte_rml_ofi.ofi_prov[cur_ofi_prov].progress_ev_active = true;
/** update the number of ofi_provs in the ofi_prov[] array **/
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"%s:%d ofi_prov id - %d created ",__FILE__,__LINE__,orte_rml_ofi.ofi_prov_open_num);
orte_rml_ofi.ofi_prov_open_num++;
}
if (fabric_info != NULL && orte_rml_ofi.ofi_prov_open_num >= MAX_OFI_PROVIDERS ) {
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"%s:%d fi_getinfo list not fully parsed as MAX_OFI_PROVIDERS - %d reached ",__FILE__,__LINE__,orte_rml_ofi.ofi_prov_open_num);
}
}
/**
* Free providers info since it's not needed anymore.
*/
fi_freeinfo(hints);
hints = NULL;
/* check if atleast one ofi_prov was successfully opened */
if (0 < orte_rml_ofi.ofi_prov_open_num ) {
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"%s:%d ofi providers openened=%d returning orte_rml_ofi.api",
__FILE__,__LINE__,orte_rml_ofi.ofi_prov_open_num);
OBJ_CONSTRUCT(&orte_rml_ofi.recv_msg_queue_list,opal_list_t);
} else {
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"%s:%d Failed to open any OFI Providers",__FILE__,__LINE__);
}
return orte_rml_ofi.ofi_prov_open_num;
}
/* return : the ofi_prov_id that corresponds to the transport requested by the attributes
if transport is not found RML_OFI_PROV_ID_INVALID is returned.
@[in]attributes : the attributes passed in to open_conduit reg the transport requested
*/
int get_ofi_prov_id( opal_list_t *attributes)
{
int ofi_prov_id = RML_OFI_PROV_ID_INVALID, prov_num=0;
char *provider = NULL, *transport = NULL;
char *ethernet="sockets", *fabric="psm2";
struct fi_info *cur_fi;
/* check the list of attributes to see if we should respond
* Attribute should have ORTE_RML_TRANSPORT_ATTRIB key
* with values "ethernet" or "fabric"
* (or) ORTE_RML_OFI_PROV_NAME key with values "socket" or "OPA"
* if both above attributes are missing return failure
*/
if (orte_get_attribute(attributes, ORTE_RML_TRANSPORT_ATTRIB, (void**)&transport, OPAL_STRING) ) {
if( 0 == strcmp( transport, "ethernet") ) {
provider = ethernet;
} else if ( 0 == strcmp( transport, "fabric") ) {
provider = fabric;
}
}
/* if from the transport we don't know which provider we want, then check for the ORTE_RML_OFI_PROV_NAME_ATTRIB */
if ( NULL == provider) {
orte_get_attribute(attributes, ORTE_RML_PROVIDER_ATTRIB, (void**)&provider, OPAL_STRING);
}
if (NULL != provider)
{
// loop the orte_rml_ofi.ofi_provs[] and find the provider name that matches
for ( prov_num = 0; prov_num < orte_rml_ofi.ofi_prov_open_num && ofi_prov_id == RML_OFI_PROV_ID_INVALID ; prov_num++ ) {
cur_fi = orte_rml_ofi.ofi_prov[prov_num].fabric_info;
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - get_ofi_prov_id() -> comparing %s = %s ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),provider,cur_fi->fabric_attr->prov_name);
if ( strcmp(provider,cur_fi->fabric_attr->prov_name) == 0) {
ofi_prov_id = prov_num;
}
}
}
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - get_ofi_prov_id(), returning ofi_prov_id=%d ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),ofi_prov_id);
return ofi_prov_id;
}
/*
* Allocate a new module and initialise ofi_prov information
* for the requested provider and return the module *
*/
static orte_rml_base_module_t* make_module( int ofi_prov_id)
{
orte_rml_ofi_module_t *mod = NULL;
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - rml_ofi make_module() begin ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
if ( RML_OFI_PROV_ID_INVALID == ofi_prov_id) {
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - open_conduit did not select any ofi provider, returning NULL ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return NULL;
}
/* create a new module */
mod = (orte_rml_ofi_module_t*)calloc(1,sizeof(orte_rml_ofi_module_t));
if (NULL == mod) {
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - Module allocation failed, returning NULL ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return NULL;
}
/* copy the APIs over to it and the OFI provider information */
memcpy(mod, &orte_rml_ofi, sizeof(orte_rml_ofi_module_t));
/* setup the remaining data locations in mod, associate conduit with ofi provider selected*/
mod->cur_transport_id = ofi_prov_id;
return (orte_rml_base_module_t*)mod;
}
/* Order of attributes honoring *
* ORTE_RML_INCLUDE_COMP_ATTRIB *
* ORTE_RML_EXCLUDE_COMP_ATTRIB *
* ORTE_RML_TRANSPORT_ATTRIB *
* ORTE_RML_PROVIDER_ATTRIB */
static orte_rml_base_module_t* open_conduit(opal_list_t *attributes)
{
char *comp_attrib = NULL;
char **comps;
int i;
orte_attribute_t *attr;
opal_list_t provider;
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - Entering rml_ofi_open_conduit()",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* Open all ofi endpoints */
if (!init_done) {
rml_ofi_component_init();
init_done = true;
}
/* check if atleast 1 ofi provider is initialised */
if ( 0 >= orte_rml_ofi.ofi_prov_open_num) {
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - Init did not open any Ofi endpoints, returning NULL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
return NULL;
}
/* someone may require this specific component, so look for "ofi" */
if (orte_get_attribute(attributes, ORTE_RML_INCLUDE_COMP_ATTRIB, (void**)&comp_attrib, OPAL_STRING) &&
NULL != comp_attrib) {
/* they specified specific components - could be multiple */
comps = opal_argv_split(comp_attrib, ',');
for (i=0; NULL != comps[i]; i++) {
if (0 == strcmp(comps[i], "ofi")) {
/* we are a candidate, */
opal_argv_free(comps);
return make_module(get_ofi_prov_id(attributes));
}
}
/* we are not a candidate */
opal_argv_free(comps);
return NULL;
} else if (orte_get_attribute(attributes, ORTE_RML_EXCLUDE_COMP_ATTRIB, (void**)&comp_attrib, OPAL_STRING) &&
NULL != comp_attrib) {
/* see if we are on the list */
comps = opal_argv_split(comp_attrib, ',');
for (i=0; NULL != comps[i]; i++) {
if (0 == strcmp(comps[i], "ofi")) {
/* we cannot be a candidate */
opal_argv_free(comps);
return NULL;
}
}
}
/*[Debug] to check for daemon commn over ofi-ethernet, enable the default conduit ORTE_MGMT_CONDUIT over ofi */
if (orte_get_attribute(attributes, ORTE_RML_TRANSPORT_TYPE, (void**)&comp_attrib, OPAL_STRING) &&
NULL != comp_attrib) {
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - Forcibly returning ofi socket provider for ethernet transport request",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
comps = opal_argv_split(comp_attrib, ',');
for (i=0; NULL != comps[i]; i++) {
if (0 == strcmp(comps[i], "ethernet")) {
/* we are a candidate, */
opal_argv_free(comps);
OBJ_CONSTRUCT(&provider, opal_list_t);
orte_set_attribute(&provider, ORTE_RML_PROVIDER_ATTRIB,
ORTE_ATTR_LOCAL, "sockets", OPAL_STRING);
return make_module(get_ofi_prov_id(&provider));
}
}
opal_argv_free(comps);
}
/*[Debug] */
/* Alternatively, check the attributes to see if we qualify - we only handle
* "pt2pt" */
OPAL_LIST_FOREACH(attr, attributes, orte_attribute_t) {
/* [TODO] add any additional attributes check here */
}
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - ofi is not a candidate as per attributes, returning NULL",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* if we get here, we cannot handle it */
return NULL;
}
static void pr_cons(orte_rml_ofi_peer_t *ptr)
{
ptr->ofi_ep = NULL;
ptr->ofi_ep_len = 0;
}
static void pr_des(orte_rml_ofi_peer_t *ptr)
{
if ( 0 < ptr->ofi_ep_len)
free( ptr->ofi_ep);
}
OBJ_CLASS_INSTANCE(orte_rml_ofi_peer_t,
opal_object_t,
pr_cons, pr_des);
/* The returned string will be of format - */
/* "<process-name>;ofi-socket:<addr_format,ip,portaddr>;ofi-<provider2>:<prov2epname>" */
/* caller will take care of string length check to not exceed limit */
static char* ofi_get_contact_info(void)
{
char *turi, *final=NULL, *tmp, *addrtype;
int rc=ORTE_SUCCESS, cur_ofi_prov=0;
struct sockaddr_in* ep_sockaddr;
/* start with our process name */
if (ORTE_SUCCESS != (rc = orte_util_convert_process_name_to_string(&final, ORTE_PROC_MY_NAME))) {
/* [TODO] ORTE_ERROR_LOG(rc); */
return final;
}
/* The returned string will be of format - "<process-name>;ofi-addr:<sin_family,sin_addr,sin_port>;" */
/* we are sending only the ethernet address */
for( cur_ofi_prov=0; cur_ofi_prov < orte_rml_ofi.ofi_prov_open_num ; cur_ofi_prov++ ) {
if ( FI_SOCKADDR_IN == orte_rml_ofi.ofi_prov[cur_ofi_prov].fabric_info->addr_format) {
ep_sockaddr = (struct sockaddr_in*)orte_rml_ofi.ofi_prov[cur_ofi_prov].ep_name;
asprintf(&addrtype, OFIADDR);
asprintf(&turi,"%d,%s,%d",ep_sockaddr->sin_family,inet_ntoa(ep_sockaddr->sin_addr),ntohs(ep_sockaddr->sin_port));
opal_output_verbose(20,orte_rml_base_framework.framework_output,
"%s - cur_ofi_prov = %d, addrtype = %s ", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),cur_ofi_prov,addrtype);
/* Add to the final string - the ofi addrtype and the epname */
asprintf(&tmp, "%s;%s:%s", final,addrtype, turi);
free(addrtype);
free(turi);
free(final);
final = tmp;
}
}
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"[%s] get_contact_info returns string - %s ", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),final);
return final;
}
static void ofi_set_contact_info (const char *uri)
{
char *uris;
opal_output_verbose(5, orte_rml_base_framework.framework_output,
"%s: OFI set_contact_info to uri %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
(NULL == uri) ? "NULL" : uri);
/* if the request doesn't contain a URI, then we
* have an error
*/
if (NULL == uri) {
opal_output(0, "%s: NULL URI", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
/* [TODO] ORTE_FORCED_TERMINATE(1);*/
return;
}
uris = strdup(uri);
process_uri(uris);
free(uris);
return;
}
static void process_uri( char *uri)
{
orte_process_name_t peer;
char *cptr, *ofiuri;
char **uris=NULL;
int rc, i=0, tot_reqd = 1, tot_found = 0;
uint64_t ui64;
orte_rml_ofi_peer_t *pr;
struct sockaddr_in* ep_sockaddr;
/* find the first semi-colon in the string */
cptr = strchr(uri, ';');
if (NULL == cptr) {
/* got a problem - there must be at least two fields,
* the first containing the process name of our peer
* and all others containing the OOB contact info
*/
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
return;
}
*cptr = '\0';
cptr++;
/* the first field is the process name, so convert it */
orte_util_convert_string_to_process_name(&peer, uri);
/* if the peer is us, no need to go further as we already
* know our own contact info
*/
if (peer.jobid == ORTE_PROC_MY_NAME->jobid &&
peer.vpid == ORTE_PROC_MY_NAME->vpid) {
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s:OFI set_contact_info peer %s is me",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&peer));
//skip adding to hashtable for HNP
if (!ORTE_PROC_IS_HNP) {
return;
} else {
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s:OFI set_contact_info - HNP process so proceeding to add to hashtable",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME) );
}
}
/* split the rest of the uri into component parts */
uris = opal_argv_split(cptr, ';');
/* get the peer object for this process */
memcpy(&ui64, (char*)&peer, sizeof(uint64_t));
if (OPAL_SUCCESS != opal_hash_table_get_value_uint64(&orte_rml_ofi.peers,
ui64, (void**)&pr) ||
NULL == pr) {
pr = OBJ_NEW(orte_rml_ofi_peer_t);
/* populate the peer object with the ofi addresses */
for(i=0; NULL != uris[i] && tot_found < tot_reqd; i++) {
ofiuri = strdup(uris[i]);
if (NULL == ofiuri) {
opal_output_verbose(2, orte_rml_base_framework.framework_output,
"%s rml:ofi: out of memory",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
continue;
}
/* Handle the OFI address types in the uri - OFIADDR(ofiaddr) */
if (0 == strncmp(ofiuri, OFIADDR, strlen(OFIADDR)) ) {
/* allocate and initialise the peer object to be inserted in hashtable */
pr->ofi_ep_len = sizeof(struct sockaddr_in);
ep_sockaddr = malloc( sizeof ( struct sockaddr_in) );
/* ofiuri for socket provider is of format - ofi-socket:<sin_family,sin_addr,sin_port> */
convert_to_sockaddr(ofiuri, ep_sockaddr);
pr->ofi_ep = (void *)ep_sockaddr;
tot_found++;
}
free( ofiuri);
}
/* if atleast one OFI address is known for peer insert it */
if( 1 <= tot_found ) {
if (OPAL_SUCCESS !=
(rc = opal_hash_table_set_value_uint64(&orte_rml_ofi.peers, ui64, (void*)pr))) {
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s: ofi peer address insertion failed for peer %s ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&peer));
ORTE_ERROR_LOG(rc);
}
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s: ofi peer address inserted for peer %s ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&peer));
opal_output_verbose(15, orte_rml_base_framework.framework_output,
"%s: ofi sock address length = %zd ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
pr->ofi_ep_len);
struct sockaddr_in* ep_sockaddr = (struct sockaddr_in*)pr->ofi_ep;
opal_output_verbose(15,orte_rml_base_framework.framework_output,
"%s OFI set_name() port = 0x%x, InternetAddr = %s ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),ntohs(ep_sockaddr->sin_port),inet_ntoa(ep_sockaddr->sin_addr));
}
}
opal_output_verbose(10,orte_rml_base_framework.framework_output,
"%s OFI end of set_contact_info()",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
opal_argv_free(uris);
return;
}
/* converts the socket uri returned by get_contact_info into sockaddr_in */
void convert_to_sockaddr( char *ofiuri, struct sockaddr_in* ep_sockaddr)
{
char *tmp, *sin_fly, *sin_port, *sin_addr;
short port;
tmp = strchr(ofiuri,':');
sin_fly = tmp+1;
tmp = strchr(sin_fly,',');
sin_addr = tmp+1;
*tmp = '\0';
tmp = strchr(sin_addr,',');
sin_port = tmp + 1;
*tmp = '\0';
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"%s OFI convert_to_sockaddr uri strings got -> family = %s, InternetAddr = %s, port = %s ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),sin_fly,sin_addr, sin_port);
ep_sockaddr->sin_family = atoi( sin_fly );
port = atoi( sin_port);
ep_sockaddr->sin_port = htons(port);
opal_output_verbose(1,orte_rml_base_framework.framework_output,
"%s OFI convert_to_sockaddr() port = 0x%x decimal-%d, InternetAddr = %s ",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),ntohs(ep_sockaddr->sin_port),ntohs(ep_sockaddr->sin_port),
inet_ntoa(ep_sockaddr->sin_addr));
}