attempt to move posting of buffers out of critical path
This commit was SVN r6469.
Этот коммит содержится в:
Tim Woodall
родитель
c54bddde5f
Коммит
262cda14cf
@ -341,7 +341,6 @@ mca_btl_base_descriptor_t* mca_btl_gm_prepare_src(
|
||||
OBJ_RETAIN(registration);
|
||||
}
|
||||
frag->registration = registration;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
@ -650,6 +649,7 @@ int mca_btl_gm_send(
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
/* post the send request */
|
||||
gm_send_with_callback(
|
||||
gm_btl->port,
|
||||
frag->hdr,
|
||||
@ -660,6 +660,15 @@ int mca_btl_gm_send(
|
||||
endpoint->endpoint_addr.port_id,
|
||||
mca_btl_gm_send_callback,
|
||||
frag);
|
||||
|
||||
if(opal_list_get_size(&gm_btl->gm_repost)) {
|
||||
mca_btl_gm_frag_t* frag;
|
||||
OPAL_THREAD_LOCK(&btl->gm_lock);
|
||||
while(NULL != (frag = (mca_btl_gm_frag_t*)opal_list_remove_first(&gm_btl->gm_repost))) {
|
||||
gm_provide_receive_buffer(gm_btl->port, frag->hdr, frag->size, frag->priority);
|
||||
}
|
||||
OPAL_THREAD_UNLOCK(&btl->gm_lock);
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
@ -57,6 +57,7 @@ struct mca_btl_gm_component_t {
|
||||
size_t gm_max_ports; /**< maximum number of ports per board */
|
||||
size_t gm_max_boards; /**< maximum number of boards */
|
||||
size_t gm_num_high_priority; /**< number of receive descriptors at high priority */
|
||||
size_t gm_num_repost;
|
||||
size_t gm_eager_frag_size;
|
||||
size_t gm_max_frag_size;
|
||||
char* gm_port_name;
|
||||
@ -97,9 +98,11 @@ struct mca_btl_gm_module_t {
|
||||
unsigned int gm_max_send_tokens;
|
||||
unsigned int gm_num_recv_tokens;
|
||||
unsigned int gm_max_recv_tokens;
|
||||
unsigned int gm_num_repost;
|
||||
|
||||
/* lock for accessing module state */
|
||||
opal_list_t gm_pending; /**< list of pending send descriptors */
|
||||
opal_list_t gm_repost; /**< list of pending fragments */
|
||||
opal_mutex_t gm_lock;
|
||||
struct mca_mpool_base_module_t* gm_mpool;
|
||||
};
|
||||
|
||||
@ -130,6 +130,8 @@ int mca_btl_gm_component_open(void)
|
||||
mca_btl_gm_param_register_int("max_modules", 4);
|
||||
mca_btl_gm_component.gm_num_high_priority =
|
||||
mca_btl_gm_param_register_int("num_high_priority", 8);
|
||||
mca_btl_gm_component.gm_num_repost =
|
||||
mca_btl_gm_param_register_int("num_repost", 4);
|
||||
mca_btl_gm_component.gm_port_name=
|
||||
mca_btl_gm_param_register_string("port_name", "OMPI");
|
||||
|
||||
@ -203,6 +205,7 @@ mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
|
||||
OBJ_CONSTRUCT(&btl->gm_frag_max, ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&btl->gm_frag_user, ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&btl->gm_pending, opal_list_t);
|
||||
OBJ_CONSTRUCT(&btl->gm_repost, opal_list_t);
|
||||
OBJ_CONSTRUCT(&btl->gm_lock, opal_mutex_t);
|
||||
|
||||
/* query nic tokens */
|
||||
@ -216,6 +219,11 @@ mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
|
||||
if(num_high_priority > (btl->gm_num_recv_tokens >> 1)) {
|
||||
num_high_priority = btl->gm_num_recv_tokens >> 1;
|
||||
}
|
||||
if(mca_btl_gm_component.gm_num_repost > num_high_priority >> 1) {
|
||||
btl->gm_num_repost = (num_high_priority >> 1);
|
||||
} else {
|
||||
btl->gm_num_repost = mca_btl_gm_component.gm_num_repost;
|
||||
}
|
||||
|
||||
/* initialize memory pool */
|
||||
resources.port = btl->port;
|
||||
@ -265,6 +273,7 @@ mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
|
||||
frag->base.des_src_cnt = 0;
|
||||
frag->base.des_dst = &frag->segment;
|
||||
frag->base.des_dst_cnt = 1;
|
||||
frag->priority = GM_LOW_PRIORITY;
|
||||
gm_provide_receive_buffer(btl->port, frag->hdr, frag->size, GM_LOW_PRIORITY);
|
||||
}
|
||||
|
||||
@ -278,6 +287,7 @@ mca_btl_gm_module_init (mca_btl_gm_module_t * btl)
|
||||
frag->base.des_src_cnt = 0;
|
||||
frag->base.des_dst = &frag->segment;
|
||||
frag->base.des_dst_cnt = 1;
|
||||
frag->priority = GM_LOW_PRIORITY;
|
||||
gm_provide_receive_buffer(btl->port, frag->hdr, frag->size, GM_LOW_PRIORITY);
|
||||
}
|
||||
|
||||
@ -495,7 +505,7 @@ int mca_btl_gm_component_progress()
|
||||
frag->segment.seg_len = gm_ntohl(event->recv.length) - sizeof(mca_btl_base_header_t);
|
||||
reg = &btl->gm_reg[hdr->tag];
|
||||
reg->cbfunc(&btl->super, hdr->tag, &frag->base, reg->cbdata);
|
||||
gm_provide_receive_buffer(btl->port, buffer, frag->size, priority);
|
||||
MCA_BTL_GM_FRAG_POST(btl,frag);
|
||||
count++;
|
||||
break;
|
||||
}
|
||||
@ -511,7 +521,7 @@ int mca_btl_gm_component_progress()
|
||||
frag->segment.seg_len = gm_ntohl(event->recv.length) - sizeof(mca_btl_base_header_t);
|
||||
reg = &btl->gm_reg[hdr->tag];
|
||||
reg->cbfunc(&btl->super, hdr->tag, &frag->base, reg->cbdata);
|
||||
gm_provide_receive_buffer(btl->port, buffer, frag->size, priority);
|
||||
MCA_BTL_GM_FRAG_POST(btl,frag);
|
||||
count++;
|
||||
break;
|
||||
}
|
||||
|
||||
@ -40,6 +40,7 @@ struct mca_btl_gm_frag_t {
|
||||
struct mca_mpool_base_registration_t* registration;
|
||||
mca_btl_base_header_t *hdr;
|
||||
size_t size;
|
||||
enum gm_priority priority;
|
||||
};
|
||||
typedef struct mca_btl_gm_frag_t mca_btl_gm_frag_t;
|
||||
OBJ_CLASS_DECLARATION(mca_btl_gm_frag_t);
|
||||
@ -105,6 +106,22 @@ OBJ_CLASS_DECLARATION(mca_btl_gm_frag_user_t);
|
||||
(opal_list_item_t*)(frag)); \
|
||||
}
|
||||
|
||||
|
||||
#define MCA_BTL_GM_FRAG_POST(btl,frag) \
|
||||
do { \
|
||||
if(opal_list_get_size(&btl->gm_repost) < btl->gm_num_repost) { \
|
||||
OPAL_THREAD_LOCK(&btl->gm_lock); \
|
||||
opal_list_append(&btl->gm_repost, (opal_list_item_t*)frag); \
|
||||
OPAL_THREAD_UNLOCK(&btl->gm_lock); \
|
||||
} else { \
|
||||
OPAL_THREAD_LOCK(&btl->gm_lock); \
|
||||
do { \
|
||||
gm_provide_receive_buffer(btl->port, frag->hdr, frag->size, priority); \
|
||||
} while (NULL != (frag = (mca_btl_gm_frag_t*)opal_list_remove_first(&btl->gm_repost))); \
|
||||
OPAL_THREAD_UNLOCK(&btl->gm_lock); \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user