ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

The IB proc/peer interfaces updated and modified.

Просмотр исходного кода 
This commit was SVN r2047.
Этот коммит содержится в:
Sayantan Sur 2004-08-11 16:45:40 +00:00
родитель 4a164fd164
Коммит 1dfc06736f
6 изменённых файлов: 131 добавлений и 105 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
src/mca/ptl/ib/src/Makefile.am
Просмотреть файл

@ -10,4 +10,5 @@ libmca_ptl_ib_la_SOURCES = \
ptl_ib.h \
ptl_ib_component.c \
ptl_ib_priv.c \
ptl_ib_proc.c
ptl_ib_proc.c \
ptl_ib_peer.c

68
src/mca/ptl/ib/src/ptl_ib.c
Просмотреть файл

@ -52,35 +52,26 @@ OBJ_CLASS_INSTANCE(mca_ptl_ib_send_request_t,
mca_pml_base_send_request_t,
NULL, NULL);
OBJ_CLASS_INSTANCE(mca_ptl_ib_peer_t,
ompi_list_item_t,
NULL, NULL);
OBJ_CLASS_INSTANCE(mca_ptl_ib_proc_t,
ompi_list_item_t,
NULL, NULL);
int mca_ptl_ib_add_procs(
struct mca_ptl_base_module_t* ptl,
size_t nprocs,
struct ompi_proc_t **ompi_procs,
struct mca_ptl_base_peer_t** peers,
ompi_bitmap_t* reachable)
int mca_ptl_ib_add_procs(struct mca_ptl_base_module_t* base_module,
size_t nprocs, struct ompi_proc_t **ompi_procs,
struct mca_ptl_base_peer_t** peers, ompi_bitmap_t* reachable)
{
int i;
int i, rc;
struct ompi_proc_t* ompi_proc;
mca_ptl_ib_proc_t* ptl_proc;
mca_ptl_base_peer_t* ptl_peer;
mca_ptl_ib_proc_t* module_proc;
mca_ptl_base_peer_t* module_peer;
D_PRINT("Adding %d procs\n", nprocs);
for(i = 0; i < nprocs; i++) {
ompi_proc = ompi_procs[i];
ptl_proc = mca_ptl_ib_proc_create(ompi_proc);
module_proc = mca_ptl_ib_proc_create(ompi_proc);
if(NULL == ptl_proc) {
if(NULL == module_proc) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
@ -90,9 +81,9 @@ int mca_ptl_ib_add_procs(
* don't bind this PTL instance to the proc.
*/
OMPI_THREAD_LOCK(&ptl_proc->proc_lock);
if(ptl_proc->proc_addr_count == ptl_proc->proc_peer_count) {
OMPI_THREAD_UNLOCK(&ptl_proc->proc_lock);
OMPI_THREAD_LOCK(&module_proc->proc_lock);
if(module_proc->proc_addr_count == module_proc->proc_peer_count) {
OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
return OMPI_ERR_UNREACH;
}
@ -100,26 +91,25 @@ int mca_ptl_ib_add_procs(
* instances that are trying to reach this destination.
* Cache the peer instance on the ptl_proc.
*/
ptl_peer = OBJ_NEW(mca_ptl_ib_peer_t);
module_peer = OBJ_NEW(mca_ptl_ib_peer_t);
if(NULL == ptl_peer) {
OMPI_THREAD_UNLOCK(&ptl_proc->proc_lock);
if(NULL == module_peer) {
OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
return OMPI_ERR_OUT_OF_RESOURCE;
}
ptl_peer->peer_ptl = (mca_ptl_ib_module_t*)ptl;
module_peer->peer_module = (mca_ptl_ib_module_t*)base_module;
/*
rc = mca_ptl_ib_proc_insert(ptl_proc, ptl_peer);
rc = mca_ptl_ib_proc_insert(module_proc, module_peer);
if(rc != OMPI_SUCCESS) {
OBJ_RELEASE(ptl_peer);
OMPI_THREAD_UNLOCK(&ptl_proc->proc_lock);
OBJ_RELEASE(module_peer);
OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
return rc;
}
*/
ompi_bitmap_set_bit(reachable, i);
OMPI_THREAD_UNLOCK(&ptl_proc->proc_lock);
peers[i] = ptl_peer;
OMPI_THREAD_UNLOCK(&module_proc->proc_lock);
peers[i] = module_peer;
}
return OMPI_SUCCESS;
}
@ -130,14 +120,14 @@ int mca_ptl_ib_del_procs(struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t ** peers)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
D_PRINT("Stub\n");
return OMPI_SUCCESS;
}
int mca_ptl_ib_finalize(struct mca_ptl_base_module_t* ptl)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
D_PRINT("Stub\n");
return OMPI_SUCCESS;
}
@ -145,7 +135,7 @@ int mca_ptl_ib_request_alloc(struct mca_ptl_base_module_t* ptl,
struct mca_pml_base_send_request_t** request)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
D_PRINT("Stub\n");
return OMPI_SUCCESS;
}
@ -154,7 +144,7 @@ void mca_ptl_ib_request_return(struct mca_ptl_base_module_t* ptl,
struct mca_pml_base_send_request_t* request)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
D_PRINT("Stub\n");
}
/*
@ -173,7 +163,7 @@ int mca_ptl_ib_send(
int flags)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
D_PRINT("Stub\n");
return OMPI_SUCCESS;
}
@ -187,6 +177,6 @@ void mca_ptl_ib_matched(
mca_ptl_base_module_t* ptl,
mca_ptl_base_recv_frag_t* frag)
{
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
/* Stub */
D_PRINT("Stub\n");
}

1
src/mca/ptl/ib/src/ptl_ib.h
Просмотреть файл

@ -44,6 +44,7 @@ struct mca_ptl_ib_component_t {
int ib_free_list_inc; /**< number of elements to alloc when growing free lists */
ompi_free_list_t ib_send_requests; /**< free list of ib send requests -- sendreq + IB */
ompi_free_list_t ib_recv_frags; /**< free list of ib recv fragments */
ompi_list_t ib_procs; /**< list of ib proc structures */
ompi_event_t ib_send_event; /**< event structure for sends */
ompi_event_t ib_recv_event; /**< event structure for recvs */
ompi_mutex_t ib_lock; /**< lock for accessing module state */

7
src/mca/ptl/ib/src/ptl_ib_component.c
Просмотреть файл

@ -91,6 +91,9 @@ static inline int mca_ptl_ib_param_register_int(
int mca_ptl_ib_component_open(void)
{
D_PRINT("Opening InfiniBand component ...\n");
OBJ_CONSTRUCT(&mca_ptl_ib_component.ib_procs, ompi_list_t);
/* register super component parameters */
mca_ptl_ib_module.super.ptl_exclusivity =
mca_ptl_ib_param_register_int ("exclusivity", 0);
@ -390,7 +393,7 @@ mca_ptl_base_module_t** mca_ptl_ib_component_init(int *num_ptl_modules,
int mca_ptl_ib_component_control(int param, void* value, size_t size)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
D_PRINT("Stub\n");
return OMPI_SUCCESS;
}
@ -402,6 +405,6 @@ int mca_ptl_ib_component_control(int param, void* value, size_t size)
int mca_ptl_ib_component_progress(mca_ptl_tstamp_t tstamp)
{
/* Stub */
fprintf(stderr,"[%s][%d]\n", __FILE__, __LINE__);
D_PRINT("Stub\n");
return OMPI_SUCCESS;
}

30
src/mca/ptl/ib/src/ptl_ib_peer.h
Просмотреть файл

@ -13,10 +13,11 @@
*/
typedef enum {
MCA_PTL_IB_CLOSED,
MCA_PTL_IB_NOT_OPENED,
MCA_PTL_IB_CONNECTING,
MCA_PTL_IB_CONNECT_ACK,
MCA_PTL_IB_CONNECTED,
MCA_PTL_IB_CLOSED,
MCA_PTL_IB_FAILED
} mca_ptl_ib_state_t;
@ -28,20 +29,19 @@ typedef enum {
*/
struct mca_ptl_base_peer_t {
ompi_list_item_t super;
struct mca_ptl_ib_module_t* peer_ptl; /**< PTL instance that created this connection */
struct mca_ptl_ib_proc_t* peer_proc; /**< proc structure corresponding to peer */
struct mca_ptl_ib_addr_t* peer_addr; /**< address of peer */
int peer_sd; /**< socket connection to peer */
mca_ptl_ib_send_frag_t* peer_send_frag; /**< current send frag being processed */
mca_ptl_ib_recv_frag_t* peer_recv_frag; /**< current recv frag being processed */
mca_ptl_ib_state_t peer_state; /**< current state of the connection */
size_t peer_retries; /**< number of connection retries attempted */
ompi_list_t peer_frags; /**< list of pending frags to send */
ompi_mutex_t peer_send_lock; /**< lock for concurrent access to peer state */
ompi_mutex_t peer_recv_lock; /**< lock for concurrent access to peer state */
ompi_event_t peer_send_event; /**< event for async processing of send frags */
ompi_event_t peer_recv_event; /**< event for async processing of recv frags */
ompi_list_item_t super;
struct mca_ptl_ib_module_t* peer_module; /**< PTL instance that created this connection */
struct mca_ptl_ib_proc_t* peer_proc; /**< proc structure corresponding to peer */
struct mca_ptl_ib_ud_addr_t* peer_addr; /**< address of peer */
mca_ptl_ib_send_frag_t* peer_send_frag; /**< current send frag being processed */
mca_ptl_ib_recv_frag_t* peer_recv_frag; /**< current recv frag being processed */
mca_ptl_ib_state_t peer_state; /**< current state of the connection */
size_t peer_retries; /**< number of connection retries attempted */
ompi_list_t peer_frags; /**< list of pending frags to send */
ompi_mutex_t peer_send_lock; /**< lock for concurrent access to peer state */
ompi_mutex_t peer_recv_lock; /**< lock for concurrent access to peer state */
ompi_event_t peer_send_event; /**< event for async processing of send frags */
ompi_event_t peer_recv_event; /**< event for async processing of recv frags */
};
typedef struct mca_ptl_base_peer_t mca_ptl_base_peer_t;
typedef struct mca_ptl_base_peer_t mca_ptl_ib_peer_t;

127
src/mca/ptl/ib/src/ptl_ib_proc.c
Просмотреть файл

@ -6,29 +6,72 @@
#include "ptl_ib_vapi.h"
#include "ptl_ib_proc.h"
static void mca_ptl_ib_proc_construct(mca_ptl_ib_proc_t* proc);
static void mca_ptl_ib_proc_destruct(mca_ptl_ib_proc_t* proc);
OBJ_CLASS_INSTANCE(mca_ptl_ib_proc_t,
ompi_list_item_t, mca_ptl_ib_proc_construct,
mca_ptl_ib_proc_destruct);
void mca_ptl_ib_proc_construct(mca_ptl_ib_proc_t* proc)
{
proc->proc_ompi = 0;
proc->proc_addrs = 0;
proc->proc_addr_count = 0;
proc->proc_peers = 0;
proc->proc_peer_count = 0;
OBJ_CONSTRUCT(&proc->proc_lock, ompi_mutex_t);
/* add to list of all proc instance */
OMPI_THREAD_LOCK(&mca_ptl_ib_component.ib_lock);
ompi_list_append(&mca_ptl_ib_component.ib_procs, &proc->super);
OMPI_THREAD_UNLOCK(&mca_ptl_ib_component.ib_lock);
}
/*
* Cleanup ib proc instance
*/
void mca_ptl_ib_proc_destruct(mca_ptl_ib_proc_t* proc)
{
/* remove from list of all proc instances */
OMPI_THREAD_LOCK(&mca_ptl_ib_component.ib_lock);
ompi_list_remove_item(&mca_ptl_ib_component.ib_procs, &proc->super);
OMPI_THREAD_UNLOCK(&mca_ptl_ib_component.ib_lock);
/* release resources */
if(NULL != proc->proc_peers) {
free(proc->proc_peers);
}
}
/*
* Look for an existing IB process instances based on the associated
* ompi_proc_t instance.
*/
/*
mca_ptl_ib_proc_t* mca_ptl_ib_proc_lookup_ompi(ompi_proc_t* ompi_proc)
static mca_ptl_ib_proc_t* mca_ptl_ib_proc_lookup_ompi(ompi_proc_t* ompi_proc)
{
mca_ptl_ib_proc_t* ib_proc;
OMPI_THREAD_LOCK(&mca_ptl_ib_module.ib_lock);
OMPI_THREAD_LOCK(&mca_ptl_ib_component.ib_lock);
for(ib_proc = (mca_ptl_ib_proc_t*)
ompi_list_get_first(&mca_ptl_ib_module.ib_procs);
ompi_list_get_first(&mca_ptl_ib_component.ib_procs);
ib_proc != (mca_ptl_ib_proc_t*)
ompi_list_get_end(&mca_ptl_ib_module.ib_procs);
ompi_list_get_end(&mca_ptl_ib_component.ib_procs);
ib_proc = (mca_ptl_ib_proc_t*)ompi_list_get_next(ib_proc)) {
if(ib_proc->proc_ompi == ompi_proc) {
OMPI_THREAD_UNLOCK(&mca_ptl_ib_module.ib_lock);
OMPI_THREAD_UNLOCK(&mca_ptl_ib_component.ib_lock);
return ib_proc;
}
}
OMPI_THREAD_UNLOCK(&mca_ptl_ib_module.ib_lock);
OMPI_THREAD_UNLOCK(&mca_ptl_ib_component.ib_lock);
return NULL;
}
*/
/*
* Create a IB process structure. There is a one-to-one correspondence
@ -43,27 +86,26 @@ mca_ptl_ib_proc_t* mca_ptl_ib_proc_create(ompi_proc_t* ompi_proc)
int rc;
size_t size;
mca_ptl_ib_proc_t* ptl_proc = NULL;
mca_ptl_ib_proc_t* module_proc = NULL;
/*
mca_ptl_ib_proc_t* ptl_proc =
mca_ptl_ib_proc_lookup_ompi(ompi_proc);
module_proc = mca_ptl_ib_proc_lookup_ompi(ompi_proc);
if(ptl_proc != NULL) {
return ptl_proc;
if(module_proc != NULL) {
return module_proc;
}
*/
ptl_proc = OBJ_NEW(mca_ptl_ib_proc_t);
module_proc = OBJ_NEW(mca_ptl_ib_proc_t);
/* Initialize number of peer */
ptl_proc->proc_peer_count = 0;
module_proc->proc_peer_count = 0;
ptl_proc->proc_ompi = ompi_proc;
module_proc->proc_ompi = ompi_proc;
/* build a unique identifier (of arbitrary
* size) to represent the proc */
ptl_proc->proc_guid = ompi_proc->proc_name;
module_proc->proc_guid = ompi_proc->proc_name;
D_PRINT("Creating proc for %d\n", ompi_proc->proc_name.vpid);
@ -72,37 +114,41 @@ mca_ptl_ib_proc_t* mca_ptl_ib_proc_create(ompi_proc_t* ompi_proc)
rc = mca_base_modex_recv(
&mca_ptl_ib_component.super.ptlm_version,
ompi_proc,
(void**)&ptl_proc->proc_addrs,
(void**)&module_proc->proc_addrs,
&size);
if(rc != OMPI_SUCCESS) {
ompi_output(0, "mca_ptl_ib_proc_create: mca_base_modex_recv: "
"failed with return value=%d", rc);
OBJ_RELEASE(ptl_proc);
OBJ_RELEASE(module_proc);
return NULL;
}
D_PRINT("UD q.p. obtained is: %d, Lid : %d\n",
(int)module_proc->proc_addrs[0].ud_qp,
(int)module_proc->proc_addrs[0].lid);
if(0 != (size % sizeof(mca_ptl_ib_ud_addr_t))) {
ompi_output(0, "mca_ptl_ib_proc_create: mca_base_modex_recv: "
"invalid size %d\n", size);
return NULL;
}
ptl_proc->proc_addr_count = size / sizeof(mca_ptl_ib_ud_addr_t);
module_proc->proc_addr_count = size / sizeof(mca_ptl_ib_ud_addr_t);
/* allocate space for peer array - one for
* each exported address
*/
ptl_proc->proc_peers = (mca_ptl_base_peer_t**)
malloc(ptl_proc->proc_addr_count * sizeof(mca_ptl_base_peer_t*));
module_proc->proc_peers = (mca_ptl_base_peer_t**)
malloc(module_proc->proc_addr_count * sizeof(mca_ptl_base_peer_t*));
if(NULL == ptl_proc->proc_peers) {
OBJ_RELEASE(ptl_proc);
if(NULL == module_proc->proc_peers) {
OBJ_RELEASE(module_proc);
return NULL;
}
return ptl_proc;
return module_proc;
}
@ -111,15 +157,14 @@ mca_ptl_ib_proc_t* mca_ptl_ib_proc_create(ompi_proc_t* ompi_proc)
* already held. Insert a ptl instance into the proc array and assign
* it an address.
*/
int mca_ptl_ib_proc_insert(mca_ptl_ib_proc_t* ptl_proc,
mca_ptl_base_peer_t* ptl_peer)
int mca_ptl_ib_proc_insert(mca_ptl_ib_proc_t* module_proc,
mca_ptl_base_peer_t* module_peer)
{
int i;
struct mca_ptl_ib_module_t *ptl_ib = ptl_peer->peer_ptl;
/* insert into peer array */
ptl_peer->peer_proc = ptl_proc;
ptl_proc->proc_peers[ptl_proc->proc_peer_count++] = ptl_peer;
module_peer->peer_proc = module_proc;
module_proc->proc_peers[module_proc->proc_peer_count++] = module_peer;
/*
* Look through the proc instance for an address that is on the
@ -127,26 +172,12 @@ int mca_ptl_ib_proc_insert(mca_ptl_ib_proc_t* ptl_proc,
* unused address.
*/
for(i=0; i<ptl_proc->proc_addr_count; i++) {
/*
mca_ptl_ib_ud_addr_t* peer_addr = ptl_proc->proc_addrs + i;
*/
for(i = 0; i < module_proc->proc_addr_count; i++) {
#if 0
unsigned long net1 = ptl_ib->ptl_ifaddr.sin_addr.s_addr & ptl_ib->ptl_ifmask.sin_addr.s_addr;
unsigned long net2 = peer_addr->addr_inet.s_addr & ptl_ib->ptl_ifmask.sin_addr.s_addr;
mca_ptl_ib_ud_addr_t* peer_addr = module_proc->proc_addrs + i;
if(peer_addr->addr_inuse != 0)
continue;
if(net1 == net2) {
ptl_peer->peer_addr = peer_addr;
break;
} else if(ptl_peer->peer_addr != 0)
ptl_peer->peer_addr = peer_addr;
#endif
module_peer->peer_addr = peer_addr;
}
/*
ptl_peer->peer_addr->addr_inuse++;
*/
return OMPI_SUCCESS;
}