ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Fix to allow eager limit and max send size to be any size (within resource limitations). Instead of storing the ompi_free_list_t * in the fragment, we use the frag type enum, this tells us where the frag came from and where it should return.. This could also be done in mvapi but is not a high priority moving forward..

Просмотр исходного кода 
Review by Brian, needs to hit the trunk + 1.1 release.. 

This commit was SVN r10157.
Этот коммит содержится в:
Galen Shipman 2006-06-01 02:32:18 +00:00
родитель 5163f2b296
Коммит 0344ae4ac5
5 изменённых файлов: 39 добавлений и 44 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

24
ompi/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -223,17 +223,9 @@ int mca_btl_openib_free(
btl->btl_mpool->mpool_release(btl->btl_mpool,
(mca_mpool_base_registration_t*)
frag->openib_reg);
MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag);
}
else if(frag->size == mca_btl_openib_component.max_send_size){
MCA_BTL_IB_FRAG_RETURN_MAX(btl, frag);
} else if(frag->size == mca_btl_openib_component.eager_limit){
MCA_BTL_IB_FRAG_RETURN_EAGER(btl, frag);
} else {
BTL_ERROR(("invalid descriptor"));
}
}
MCA_BTL_IB_FRAG_RETURN(((mca_btl_openib_module_t*) btl), frag);
return OMPI_SUCCESS;
}
@ -326,7 +318,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
reserve == 0) {
/* The user buffer is contigous and we are asked to send more than the max send size. */
MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc);
MCA_BTL_IB_FRAG_ALLOC_FRAG(openib_btl, frag, rc);
if(NULL == frag){
return NULL;
}
@ -348,7 +340,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
(mca_mpool_base_registration_t**) &openib_reg);
if(OMPI_SUCCESS != rc || NULL == openib_reg) {
BTL_ERROR(("mpool_register(%p,%lu) failed", iov.iov_base, max_data));
MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag);
MCA_BTL_IB_FRAG_RETURN(openib_btl, frag);
return NULL;
}
@ -386,7 +378,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
rc = ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
*size = max_data;
if( rc < 0 ) {
MCA_BTL_IB_FRAG_RETURN_EAGER(btl, frag);
MCA_BTL_IB_FRAG_RETURN(openib_btl, frag);
return NULL;
}
@ -416,7 +408,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
*size = max_data;
if( rc < 0 ) {
MCA_BTL_IB_FRAG_RETURN_MAX(btl, frag);
MCA_BTL_IB_FRAG_RETURN(openib_btl, frag);
return NULL;
}
@ -493,7 +485,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
if(OMPI_SUCCESS != rc || NULL == openib_reg) {
BTL_ERROR(("mpool_register(%p,%lu) failed: base %p lb %lu offset %lu",
frag->segment.seg_addr.pval, *size, convertor->pBaseBuf, lb, convertor->bConverted));
MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag);
MCA_BTL_IB_FRAG_RETURN(openib_btl, frag);
return NULL;
}
}

2
ompi/mca/btl/openib/btl_openib.h
Просмотреть файл

@ -57,7 +57,7 @@ struct mca_btl_openib_component_t {
int ib_max_btls;
/**< maximum number of hcas available to the IB component */
uint32_t ib_num_btls;
int ib_num_btls;
/**< number of hcas available to the IB component */
struct mca_btl_openib_module_t *openib_btls;

10
ompi/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -237,8 +237,7 @@ int mca_btl_openib_component_close(void)
static int
mca_btl_openib_modex_send(void)
{
int rc;
size_t i;
int rc, i;
size_t size;
mca_btl_openib_port_info_t *ports = NULL;
@ -322,15 +321,16 @@ mca_btl_base_module_t** mca_btl_openib_component_init(int *num_btl_modules,
bool enable_mpi_threads)
{
struct ibv_device **ib_devs;
uint32_t num_devs;
mca_btl_base_module_t** btls;
uint32_t i,j, length;
int i,j, length, num_devs;
struct mca_mpool_base_resources_t mpool_resources;
opal_list_t btl_list;
mca_btl_openib_module_t * openib_btl;
mca_btl_base_selected_module_t* ib_selected;
opal_list_item_t* item;
#if OMPI_MCA_BTL_OPENIB_HAVE_DEVICE_LIST == 0
struct dlist *dev_list;
#endif
struct ibv_device* ib_dev;
unsigned short seedv[3];
@ -718,7 +718,7 @@ int mca_btl_openib_handle_incoming_hp(
int mca_btl_openib_component_progress()
{
uint32_t i, j, c;
int i, j, c;
int count = 0,ne = 0, ret;
int32_t credits;
mca_btl_openib_frag_t* frag;

16
ompi/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -1049,7 +1049,8 @@ static void mca_btl_openib_endpoint_credits_lp(
mca_btl_openib_endpoint_send_credits_lp(endpoint);
}
}
MCA_BTL_IB_FRAG_RETURN_EAGER((mca_btl_openib_module_t*)btl, (mca_btl_openib_frag_t*)descriptor);
MCA_BTL_IB_FRAG_RETURN(((mca_btl_openib_module_t*)btl),
((mca_btl_openib_frag_t*)descriptor));
}
/**
@ -1095,7 +1096,7 @@ void mca_btl_openib_endpoint_send_credits_lp(
&bad_wr)) {
OPAL_THREAD_ADD32(&endpoint->sd_credits_lp, -1);
OPAL_THREAD_ADD32(&endpoint->rd_credits_lp, frag->hdr->credits);
MCA_BTL_IB_FRAG_RETURN_EAGER(openib_btl, frag);
MCA_BTL_IB_FRAG_RETURN(openib_btl, frag);
BTL_ERROR(("error posting send request errno %d says %s", strerror(errno)));
return;
}
@ -1126,7 +1127,8 @@ static void mca_btl_openib_endpoint_credits_hp(
mca_btl_openib_endpoint_send_credits_hp(endpoint);
}
}
MCA_BTL_IB_FRAG_RETURN_EAGER((mca_btl_openib_module_t*)btl, (mca_btl_openib_frag_t*)descriptor);
MCA_BTL_IB_FRAG_RETURN(((mca_btl_openib_module_t*)btl),
((mca_btl_openib_frag_t*)descriptor));
}
/**
@ -1176,7 +1178,7 @@ void mca_btl_openib_endpoint_send_credits_hp(
&bad_wr)) {
OPAL_THREAD_ADD32(&endpoint->sd_credits_hp, -1);
OPAL_THREAD_ADD32(&endpoint->rd_credits_hp, frag->hdr->credits);
MCA_BTL_IB_FRAG_RETURN_EAGER(openib_btl, frag);
MCA_BTL_IB_FRAG_RETURN(openib_btl, frag);
BTL_ERROR(("error posting send request errno %d says %s", errno,
strerror(errno)));
return;
@ -1189,8 +1191,8 @@ static void mca_btl_openib_endpoint_eager_rdma(
struct mca_btl_base_descriptor_t* descriptor,
int status)
{
MCA_BTL_IB_FRAG_RETURN_EAGER((mca_btl_openib_module_t*)btl,
(mca_btl_openib_frag_t*)descriptor);
MCA_BTL_IB_FRAG_RETURN(((mca_btl_openib_module_t*)btl),
((mca_btl_openib_frag_t*)descriptor));
}
static int mca_btl_openib_endpoint_send_eager_rdma(
@ -1220,7 +1222,7 @@ static int mca_btl_openib_endpoint_send_eager_rdma(
frag->segment.seg_len = sizeof(mca_btl_openib_eager_rdma_header_t);
if (mca_btl_openib_endpoint_post_send(openib_btl, endpoint, frag) !=
OMPI_SUCCESS) {
MCA_BTL_IB_FRAG_RETURN_EAGER(openib_btl, frag);
MCA_BTL_IB_FRAG_RETURN(openib_btl, frag);
BTL_ERROR(("Error sending RDMA buffer", strerror(errno)));
return -1;
}

31
ompi/mca/btl/openib/btl_openib_frag.h
Просмотреть файл

@ -135,12 +135,6 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_recv_frag_max_t);
frag = (mca_btl_openib_frag_t*) item; \
}
#define MCA_BTL_IB_FRAG_RETURN_EAGER(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_btl_openib_module_t*)btl)->send_free_eager, (opal_list_item_t*)(frag)); \
}
#define MCA_BTL_IB_FRAG_ALLOC_MAX(btl, frag, rc) \
{ \
\
@ -149,12 +143,6 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_recv_frag_max_t);
frag = (mca_btl_openib_frag_t*) item; \
}
#define MCA_BTL_IB_FRAG_RETURN_MAX(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_btl_openib_module_t*)btl)->send_free_max, (opal_list_item_t*)(frag)); \
}
#define MCA_BTL_IB_FRAG_ALLOC_FRAG(btl, frag, rc) \
{ \
\
@ -163,9 +151,22 @@ OBJ_CLASS_DECLARATION(mca_btl_openib_recv_frag_max_t);
frag = (mca_btl_openib_frag_t*) item; \
}
#define MCA_BTL_IB_FRAG_RETURN_FRAG(btl, frag) \
{ \
OMPI_FREE_LIST_RETURN(&((mca_btl_openib_module_t*)btl)->send_free_frag, (opal_list_item_t*)(frag)); \
#define MCA_BTL_IB_FRAG_RETURN(btl, frag) \
{ do { \
ompi_free_list_t* my_list; \
switch(frag->type) { \
case MCA_BTL_OPENIB_FRAG_EAGER_RDMA: \
case MCA_BTL_OPENIB_FRAG_EAGER: \
my_list = &btl->send_free_eager; \
break; \
case MCA_BTL_OPENIB_FRAG_MAX: \
my_list = &btl->send_free_max; \
break; \
case MCA_BTL_OPENIB_FRAG_FRAG: \
my_list = &btl->send_free_frag; \
} \
OMPI_FREE_LIST_RETURN(my_list, (opal_list_item_t*)(frag)); \
} while(0); \
}