1
1
This commit was SVN r2665.
Этот коммит содержится в:
Gopal Santhanaraman 2004-09-14 15:29:43 +00:00
родитель 5f25433bd3
Коммит 73ff3f22c5
25 изменённых файлов: 1400 добавлений и 271 удалений

Просмотреть файл

@ -21,8 +21,6 @@
#include "ptl_gm_peer.h"
#include "ptl_gm_priv.h"
#define DEBUG 0
mca_ptl_gm_module_t mca_ptl_gm_module = {
{
&mca_ptl_gm_component.super,
@ -40,32 +38,19 @@ mca_ptl_gm_module_t mca_ptl_gm_module = {
mca_ptl_gm_add_procs,
mca_ptl_gm_del_procs,
mca_ptl_gm_finalize,
mca_ptl_gm_send, /* JMS: Need send here */
mca_ptl_gm_send,
mca_ptl_gm_put,
mca_ptl_gm_get,
mca_ptl_gm_matched,
mca_ptl_gm_request_init, /* JMS need request init here */
mca_ptl_gm_request_fini, /* JMS need request fini here */
NULL, /* JMS need match here */
NULL, /* JMS need send_progress here */
NULL /* JMS need recv_progress here */
mca_ptl_gm_request_init,
mca_ptl_gm_request_fini,
}
};
/*OBJ_CLASS_INSTANCE (mca_ptl_gm_recv_frag_t,*/
/*mca_ptl_base_recv_frag_t, NULL, NULL);*/
OBJ_CLASS_INSTANCE (mca_ptl_gm_send_request_t,
mca_pml_base_send_request_t, NULL, NULL);
OBJ_CLASS_INSTANCE (mca_ptl_gm_peer_t, ompi_list_item_t, NULL, NULL);
/*
*
*/
int
mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
size_t nprocs,
@ -103,8 +88,6 @@ mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
/* FIXME: */
for (j=0; j < num_peer_ptls; j++) {
/*XXX: check for self */
ptl_peer = OBJ_NEW (mca_ptl_gm_peer_t);
if (NULL == ptl_peer) {
OMPI_THREAD_UNLOCK (&ptl_proc->proc_lock);
@ -120,25 +103,20 @@ mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
ptl_proc->proc_addrs[j].global_id,
&lid)) {
ompi_output (0,
"[%s:%d] error in converting global to local id \n", __FILE__, __LINE__);
"[%s:%d] error in converting global to local id \n",
__FILE__, __LINE__);
}
ptl_peer->local_id = lid;
ptl_proc->peer_arr[ptl_proc->proc_peer_count] = ptl_peer;
ptl_proc->proc_peer_count++;
ptl_peer->peer_addr = ptl_proc->proc_addrs + i;
}
ompi_bitmap_set_bit (reachable, i);
OMPI_THREAD_UNLOCK (&ptl_proc->proc_lock);
peers[i] = (struct mca_ptl_base_peer_t*)ptl_peer;
}
#if DEBUG
printf ("returning with success from gm_add_procs\n");
#endif
return OMPI_SUCCESS;
}
@ -161,7 +139,6 @@ mca_ptl_gm_del_procs (struct mca_ptl_base_module_t *ptl,
/*
*
*/
int
mca_ptl_gm_finalize (struct mca_ptl_base_module_t *ptl)
{
@ -176,6 +153,9 @@ mca_ptl_gm_request_init(struct mca_ptl_base_module_t *ptl,
mca_ptl_gm_send_frag_t *frag;
struct mca_ptl_gm_send_request_t *req;
GM_DBG(PTL_GM_DBG_COMM,
"INSIDE REQUEST INIT: the request is %p\n",request);
frag = mca_ptl_gm_alloc_send_frag(ptl, request);
if (NULL == frag)
@ -199,10 +179,8 @@ mca_ptl_gm_request_init(struct mca_ptl_base_module_t *ptl,
*/
void
mca_ptl_gm_request_fini (struct mca_ptl_base_module_t *ptl,
struct mca_pml_base_send_request_t *request)
struct mca_pml_base_send_request_t *request)
{
mca_ptl_gm_send_frag_t *frag;
frag = ((mca_ptl_gm_send_request_t *)request)->req_frag;
@ -224,9 +202,14 @@ mca_ptl_gm_send (struct mca_ptl_base_module_t *ptl,
mca_ptl_gm_module_t * gm_ptl;
int rc;
GM_DBG(PTL_GM_DBG_COMM,"INSIDE PTL GM SEND\n");
gm_ptl = (mca_ptl_gm_module_t *)ptl;
if (offset == 0) {
GM_DBG(PTL_GM_DBG_COMM,"OFFSET = 0\n");
sendfrag = ((mca_ptl_gm_send_request_t *)sendreq)->req_frag;
sendfrag->req = sendreq;
assert(sendreq != NULL);
} else {
sendfrag = mca_ptl_gm_alloc_send_frag (ptl,sendreq);
if (NULL == sendfrag) {
@ -238,8 +221,9 @@ mca_ptl_gm_send (struct mca_ptl_base_module_t *ptl,
((struct mca_ptl_gm_send_request_t *)sendreq)->req_frag =sendfrag;
((struct mca_ptl_gm_send_request_t *)sendreq)->need_ack = flags;
rc = mca_ptl_gm_send_frag_init (sendfrag, (mca_ptl_gm_peer_t*)ptl_peer, sendreq, offset,
&size, flags);
rc = mca_ptl_gm_send_frag_init (sendfrag,
(mca_ptl_gm_peer_t*)ptl_peer,
sendreq, offset, &size, flags);
/*initiate the send */
gm_ptl_peer = (mca_ptl_gm_peer_t *)ptl_peer;
@ -248,11 +232,7 @@ mca_ptl_gm_send (struct mca_ptl_base_module_t *ptl,
gm_ptl->num_send_tokens--;
/*Update offset */
sendreq->req_offset += rc; /* XXX: should be what convertor packs */
/*append to the send_fragments_queue. */
/*ompi_list_append (&(gm_ptl->gm_send_frags_queue),*/
/*(ompi_list_item_t *) sendfrag);*/
sendreq->req_offset += size; /* XXX: should be what convertor packs */
return OMPI_SUCCESS;
}
@ -280,14 +260,11 @@ mca_ptl_gm_put (struct mca_ptl_base_module_t *ptl,
destination_buffer =(void *)( (sendreq->req_peer_addr).pval);
/* register the user buffer */
if (offset > 0)
{
if (offset > 0) {
status = gm_register_memory(gm_ptl->my_port, buffer_ptr, bytes_reg);
if(GM_SUCCESS != status)
{
if(GM_SUCCESS != status) {
ompi_output(0,"[%s:%d] Unable to register memory\n",__FILE__,__LINE__);
}
}
putfrag = mca_ptl_gm_alloc_send_frag (ptl,sendreq); /*alloc_put_frag */
@ -297,31 +274,20 @@ mca_ptl_gm_put (struct mca_ptl_base_module_t *ptl,
((struct mca_ptl_gm_send_request_t *)sendreq)->req_frag =putfrag;
((struct mca_ptl_gm_send_request_t *)sendreq)->need_ack = flags;
rc = mca_ptl_gm_put_frag_init(putfrag ,
(mca_ptl_gm_peer_t*)ptl_peer,gm_ptl,
sendreq, offset, &size, flags);
/* check that we have a send token available */
rc =
mca_ptl_gm_peer_put((mca_ptl_gm_peer_t *)ptl_peer, putfrag,
rc = mca_ptl_gm_peer_put((mca_ptl_gm_peer_t *)ptl_peer, putfrag,
sendreq, offset, &size, flags,
destination_buffer, bytes_reg);
gm_ptl->num_send_tokens--;
sendreq->req_offset += size;
sendreq->req_offset += size; /* should be what is returned by put */
return OMPI_SUCCESS;
}
/*
* initiate a get.
*/
@ -335,15 +301,9 @@ mca_ptl_gm_get (struct mca_ptl_base_module_t *ptl,
return OMPI_SUCCESS;
}
/* A posted receive has been matched - if required send an
* ack back to the peer and process the fragment.
*/
void
mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
mca_ptl_base_recv_frag_t * frag )
@ -353,7 +313,7 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
mca_ptl_base_header_t *header;
int bytes_recv, rc,rc1, total_bytes, bytes_reg;
mca_ptl_gm_module_t *gm_ptl;
struct iovec iov[1];
struct iovec iov;
mca_ptl_gm_send_frag_t *ack;
mca_ptl_gm_recv_frag_t *recv_frag;
char *buffer_ptr;
@ -370,6 +330,7 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
/* might need to send an ack back */
recv_frag = (mca_ptl_gm_recv_frag_t *) frag;
ack = mca_ptl_gm_alloc_send_frag(ptl,NULL);
/*associate null request for ack*/
if (NULL == ack) {
ompi_output(0,"[%s:%d] unable to alloc a gm fragment\n",
@ -384,48 +345,43 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
{
buffer_ptr = (char *)( request->req_base.req_addr );
total_bytes = (request->req_base.req_datatype->size) *
(request->req_base.req_count);
total_bytes = request->req_bytes_packed;
/*total_bytes = (request->req_base.req_datatype->size) **/
/*(request->req_base.req_count);*/
bytes_recv = frag->frag_base.frag_size - header->hdr_common.hdr_size;
bytes_reg = total_bytes - bytes_recv;
buffer_ptr += bytes_recv;
status = gm_register_memory(gm_ptl->my_port, buffer_ptr, bytes_reg);
recv_frag->registered_buf = buffer_ptr;
printf("Receiver: register addr: %p, bytes: %d\n",buffer_ptr,bytes_reg);
fflush(stdout);
GM_DBG(PTL_GM_DBG_COMM,"Receiver: register addr: %p, bytes: %d\n",buffer_ptr,bytes_reg);
if(GM_SUCCESS != status)
{
if(GM_SUCCESS != status) {
ompi_output(0,"[%s:%d] Unable to register memory\n",__FILE__,__LINE__);
}
/* send the regiscered memory information, send recv request * ptr */
/* send the registered memory information, send recv request * ptr */
rc1 = mca_ptl_gm_send_ack_init (ack, gm_ptl,
(mca_ptl_gm_peer_t *)(recv_frag->frag_recv.frag_base.frag_peer)
, recv_frag, buffer_ptr, bytes_reg);
/*XXX : put the registered memory in pin-down cache */
/*XXX: check this*/
rc1 = mca_ptl_gm_peer_send ((mca_ptl_gm_peer_t *)
(ack->send_frag.frag_base.frag_peer),
ack,srequest,0,&size,0 );
mca_ptl_gm_peer_send (
(mca_ptl_gm_peer_t *) (ack->send_frag.frag_base.frag_peer),
ack,srequest,0,&size,0 );
GM_DBG(PTL_GM_DBG_COMM,"RECEIVER FINISHED SENDING ACK\n");
gm_ptl->num_send_tokens--;
/*ompi_list_append (&(gm_ptl->gm_send_frags_queue),*/
/*(ompi_list_item_t *) ack);*/
}
}
/* Here we expect that frag_addr is the begin of the buffer header included */
iov[0].iov_base = ((char*)frag->frag_base.frag_addr) + header->hdr_common.hdr_size;
iov.iov_base = ((char*)frag->frag_base.frag_addr) + header->hdr_common.hdr_size;
bytes_recv = frag->frag_base.frag_size - header->hdr_common.hdr_size;
iov[0].iov_len = bytes_recv;
iov.iov_len = bytes_recv;
if (header->hdr_frag.hdr_frag_length > 0) {
/* ompi_proc_t *proc;
ompi_proc_t *proc;
proc = ompi_comm_peer_lookup(request->req_base.req_comm,
request->req_base.req_peer);
@ -439,26 +395,26 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
request->req_base.req_count,
request->req_base.req_addr,
header->hdr_frag.hdr_frag_offset);
rc = ompi_convertor_unpack(&frag->frag_base.frag_convertor, &(iov[0]), 1);
assert( rc == 1 );*/
printf ("in matched: bytes received is %d\n", bytes_recv);
fflush(stdout);
memcpy(request->req_base.req_addr,iov[0].iov_base,bytes_recv);
rc = ompi_convertor_unpack(&frag->frag_base.frag_convertor, &(iov), 1);
assert( rc >= 0 );
GM_DBG(PTL_GM_DBG_COMM,"in matched: bytes received is %d\n", bytes_recv);
/*memcpy(request->req_base.req_addr,iov.iov_base,bytes_recv);*/
}
/*update progress*/
ptl->ptl_recv_progress( ptl, request, bytes_recv, iov[0].iov_len );
ptl->ptl_recv_progress( ptl, request, bytes_recv, iov.iov_len );
/* Now update the status of the fragment */
((mca_ptl_gm_recv_frag_t*)frag)->matched = true;
if( ((mca_ptl_gm_recv_frag_t*)frag)->have_allocated_buffer == true ){
free( frag->frag_base.frag_addr );
/*if( ((mca_ptl_gm_recv_frag_t*)frag)->have_allocated_buffer == true
* ){*/
/*free( frag->frag_base.frag_addr );*/
((mca_ptl_gm_recv_frag_t*)frag)->have_allocated_buffer = false;
}
/*}*/
/*return to free list */
gm_ptl = (mca_ptl_gm_module_t *)ptl;
OMPI_FREE_LIST_RETURN(&(gm_ptl->gm_recv_frags_free), (ompi_list_item_t*)frag);
OMPI_FREE_LIST_RETURN(&(gm_ptl->gm_recv_frags_free),
(ompi_list_item_t*)((mca_ptl_gm_recv_frag_t*)frag));
}

Просмотреть файл

@ -19,13 +19,13 @@
#define MCA_PTL_GM_STATISTICS 0
#define GM_SIZE 30
#define THRESHOLD 16384
#define THRESHOLD 16384
#define MAX_GM_PORTS 16
#define MAX_RECV_TOKENS 256
#define PTL_GM_ADMIN_SEND_TOKENS 0
#define PTL_GM_ADMIN_RECV_TOKENS 0
#define GM_SEND_BUF_SIZE 16384
#define GM_RECV_BUF_SIZE 16384
#define GM_RECV_BUF_SIZE 16384
#define NUM_RECV_FRAGS 100
#define MCA_PTL_GM_FRAG_CACHED

Просмотреть файл

@ -111,7 +111,8 @@ mca_ptl_gm_component_open (void)
/* register GM component parameters */
mca_ptl_gm_module.super.ptl_first_frag_size =
mca_ptl_gm_param_register_int ("first_frag_size", ((16 * 1024) - 64));
mca_ptl_gm_param_register_int ("first_frag_size",
((PTL_GM_FIRST_FRAG_SIZE) - 64));
mca_ptl_gm_module.super.ptl_min_frag_size =
mca_ptl_gm_param_register_int ("min_frag_size", 1<<16);
mca_ptl_gm_module.super.ptl_max_frag_size =
@ -231,7 +232,7 @@ ompi_mca_ptl_gm_init (mca_ptl_gm_component_t * gm)
/* open the first available gm port for this board */
board_no = i;
for (port_no = 2; port_no < MAX_GM_PORTS; port_no++) {
printf ("about to call open port\n");
GM_DBG(PTL_GM_DBG_COMM,"about to call open port\n");
if (port_no == 3) continue;
/* port 0,1,3 reserved */
status = gm_open (&(ptl->my_port), board_no,
@ -338,8 +339,7 @@ ompi_mca_ptl_gm_init_sendrecv (mca_ptl_gm_component_t * gm)
ompi_free_list_init (&(ptl->gm_recv_frags_free),
sizeof (mca_ptl_gm_recv_frag_t),
OBJ_CLASS (mca_ptl_gm_recv_frag_t),
ptl->num_recv_tokens,ptl->num_recv_tokens, 1, NULL); /* not using mpool */
ptl->num_recv_tokens,ptl->num_recv_tokens, 1, NULL);
/*allocate the elements */
free_rfragment = (mca_ptl_gm_recv_frag_t *)
@ -435,21 +435,16 @@ mca_ptl_gm_component_control (int param, void *value, size_t size)
/*
* GM module progress.
*/
int
mca_ptl_gm_component_progress (mca_ptl_tstamp_t tstamp)
{
int rc;
/* check the send queue to see if any pending send can proceed */
/* check for receive and , call ptl_match to send it to the upper
level */
/* XXX: Do all the following inside a dispatcher, either in this routine
* or mca_ptl_gm_incoming_recv(), YUW
* i) check the send queue to see if any pending send can proceed
* ii) check for recieve and , call ptl_match to send it to the upper level
* BTW, ptl_matced is invoked inside ptl_match() via PML.
*/
rc = mca_ptl_gm_incoming_recv(&mca_ptl_gm_component);
return OMPI_SUCCESS;
/* check the send queue to see if any pending send can proceed */
/* check for recieve and , call ptl_match to send it to the upper
level */
/* in case matched, do the appropriate queuing. */
}

Просмотреть файл

@ -23,8 +23,6 @@
#include "ptl_gm_sendfrag.h"
#include "ptl_gm_priv.h"
#define DEBUG 0
int mca_ptl_gm_peer_put(mca_ptl_gm_peer_t *ptl_peer,
mca_ptl_gm_send_frag_t *fragment,
struct mca_pml_base_send_request_t *sendreq,
@ -58,7 +56,7 @@ int mca_ptl_gm_peer_send(mca_ptl_gm_peer_t *ptl_peer,
size_t *size,
int flags)
{
struct iovec outvec[1];
struct iovec outvec;
size_t size_in,size_out;
int header_length;
mca_ptl_base_frag_header_t* header;
@ -68,12 +66,12 @@ int mca_ptl_gm_peer_send(mca_ptl_gm_peer_t *ptl_peer,
size_in = *size;
outvec[0].iov_base = (char*)fragment->send_buf;
outvec.iov_base = (char*)fragment->send_buf;
if( (size_in + header_length) <= GM_SEND_BUF_SIZE )
outvec[0].iov_len = size_in;
outvec.iov_len = size_in;
else
outvec[0].iov_len = GM_SEND_BUF_SIZE - header_length;
outvec.iov_len = GM_SEND_BUF_SIZE - header_length;
if(size_in > 0) {
@ -107,17 +105,18 @@ int mca_ptl_gm_peer_send(mca_ptl_gm_peer_t *ptl_peer,
/*XXX: need to add the header */
/*copy the data to the registered buffer*/
outvec[0].iov_base = ((char*)fragment->send_buf) + header_length;
outvec.iov_base = ((char*)fragment->send_buf) + header_length;
if((rc = ompi_convertor_pack(convertor, &(outvec[0]), 1)) < 0)
if((rc = ompi_convertor_pack(convertor, &(outvec), 1)) < 0)
return OMPI_ERROR;
}
/* update the fields */
outvec[0].iov_len += header_length;
outvec[0].iov_base = fragment->send_buf;
outvec.iov_len += header_length;
outvec.iov_base = fragment->send_buf;
/* adjust size and request offset to reflect actual number of bytes
* packed by convertor */
size_out = outvec[0].iov_len;
size_out = outvec.iov_len;
/* initiate the gm send */
gm_send_with_callback( ptl_peer->peer_ptl->my_port, fragment->send_buf,
@ -126,11 +125,11 @@ int mca_ptl_gm_peer_send(mca_ptl_gm_peer_t *ptl_peer,
fragment->send_frag.frag_base.frag_owner = &ptl_peer->peer_ptl->super;
fragment->send_frag.frag_base.frag_peer = (struct mca_ptl_base_peer_t*)ptl_peer;
fragment->send_frag.frag_base.frag_addr = outvec[0].iov_base;
fragment->send_frag.frag_base.frag_addr = outvec.iov_base;
fragment->send_frag.frag_base.frag_size = size_out;
return (size_out - header_length);
/*return OMPI_SUCCESS;*/
*size = (size_out - header_length);
return OMPI_SUCCESS;
}
@ -151,20 +150,13 @@ void put_callback(struct gm_port *port,void * context, gm_status_t status)
bytes = putfrag->send_frag.frag_base.frag_size;
#if DEBUG
printf("ENTERING PUT CALLBACK\n");
fflush(stdout);
#endif
GM_DBG(PTL_GM_DBG_COMM,"ENTERING PUT CALLBACK\n");
switch (status) {
case GM_SUCCESS:
/* local put completed, mark put as complete */
#if DEBUG
printf("PUTCALLBACK WITH CASE GM_SUCCESS\n");
fflush(stdout);
#endif
GM_DBG(PTL_GM_DBG_COMM,"PUTCALLBACK WITH CASE GM_SUCCESS\n");
ptl->num_send_tokens++;
putfrag->put_sent = 1;
@ -172,26 +164,18 @@ void put_callback(struct gm_port *port,void * context, gm_status_t status)
rc = mca_ptl_gm_peer_send (putfrag->peer,putfrag,send_req,
offset,&size,flags);
#if DEBUG
printf("after issuing the put completion the request offset = %d\n",send_req->req_offset);
fflush(stdout);
#endif
assert(rc == 0);
GM_DBG(PTL_GM_DBG_COMM,"FINISHED SENDING FIN\n");
GM_DBG(PTL_GM_DBG_COMM,"after issuing the put completion the request offset = %d\n",send_req->req_offset);
/* deregister the user memory */
status = gm_deregister_memory(ptl->my_port, (char *)(putfrag->registered_buf), bytes);
if(GM_SUCCESS != status)
{
#if DEBUG
ompi_output(0," unpinning memory failed\n");
#endif
}
else
{
#if DEBUG
ompi_output(0," unpinning %d bytes of memory success\n",bytes);
#endif
if(GM_SUCCESS != status) {
ompi_output(0," unpinning memory failed\n");
}
else {
GM_DBG(PTL_GM_DBG_COMM, " unpinning %d bytes of memory success\n",bytes);
}
break;
@ -228,7 +212,7 @@ void send_callback(struct gm_port *port,void * context, gm_status_t status)
header = (mca_ptl_base_header_t*)frag->send_buf;
header_length = ((mca_ptl_base_header_t*)header)->hdr_common.hdr_size;
if (frag->type == 1)
if (frag->type == PUT)
{
bytes = header->hdr_ack.hdr_dst_size;
}
@ -242,34 +226,23 @@ void send_callback(struct gm_port *port,void * context, gm_status_t status)
frag->wait_for_ack = 1;
}
switch (status) {
case GM_SUCCESS:
/* send completed, can reuse the user buffer */
#if DEBUG
printf("SENDCALLBACK WITH CASE GM_SUCCESS\n");
fflush(stdout);
#endif
GM_DBG(PTL_GM_DBG_COMM,
"SENDCALLBACK WITH CASE GM_SUCCESS: frag->wait_for_ack = %d\n",
frag->wait_for_ack);
ptl->num_send_tokens++;
frag->send_complete = 1;
if (frag->wait_for_ack == 0 && (gm_send_req != NULL))
{
#if DEBUG
printf("inside send callback : calling send progress bytes = %d\n",bytes);
fflush(stdout);
#endif
ptl->super.ptl_send_progress( (mca_ptl_base_module_t*)ptl,
gm_send_req, bytes );
if ((frag->wait_for_ack == 0) && (gm_send_req != NULL)) {
GM_DBG(PTL_GM_DBG_COMM,"inside send callback : calling send progress bytes = %d\n",bytes);
ptl->super.ptl_send_progress( (mca_ptl_base_module_t*)ptl,
gm_send_req, bytes );
}
OMPI_FREE_LIST_RETURN(&(ptl->gm_send_frags), (ompi_list_item_t *)frag);
/*list = (ompi_list_t *)(&(ptl->gm_send_frags_queue));*/
/*ompi_list_remove_first(list);*/
break;
case GM_SEND_TIMED_OUT:
@ -285,6 +258,8 @@ void send_callback(struct gm_port *port,void * context, gm_status_t status)
"[%s:%d] error in message completion\n",__FILE__,__LINE__);
break;
}
GM_DBG(PTL_GM_DBG_COMM,"RETURNING FROM SEND_CALLBACK\n");
}
@ -302,6 +277,7 @@ void ptl_gm_ctrl_frag(struct mca_ptl_gm_module_t *ptl,
{
frag = (mca_ptl_gm_send_frag_t *)header->hdr_ack.hdr_src_ptr.pval;
req = (mca_pml_base_send_request_t *) frag->req;
assert(req != NULL);
req->req_peer_match.pval = header->hdr_ack.hdr_dst_match.pval;
req->req_peer_addr.pval = header->hdr_ack.hdr_dst_addr.pval;
req->req_peer_size = header->hdr_ack.hdr_dst_size;
@ -325,14 +301,13 @@ void ptl_gm_ctrl_frag(struct mca_ptl_gm_module_t *ptl,
if(header->hdr_common.hdr_type == MCA_PTL_HDR_TYPE_FIN)
{
GM_DBG(PTL_GM_DBG_COMM,"CASE: HDR_TYPE_FIN\n");
request = (mca_pml_base_recv_request_t*)
header->hdr_ack.hdr_dst_match.pval;
/* call receive progress and indicate the recv has been completed */
#if DEBUG
printf("Calling recv_progress with bytes = %d\n",header->hdr_ack.hdr_dst_size);
fflush(stdout);
#endif
GM_DBG(PTL_GM_DBG_COMM,"Calling recv_progress with bytes = %d\n",header->hdr_ack.hdr_dst_size);
ptl->super.ptl_recv_progress (
(mca_ptl_base_module_t *) ptl,
request ,
@ -344,33 +319,26 @@ void ptl_gm_ctrl_frag(struct mca_ptl_gm_module_t *ptl,
status = gm_deregister_memory(ptl->my_port, reg_buf,
bytes);
if(GM_SUCCESS != status)
{
#if DEBUG
if(GM_SUCCESS != status) {
ompi_output(0," unpinning memory failed\n");
#endif
}
else
{
#if DEBUG
ompi_output(0,"unpinning memory success,addr:%p,bytes:%d\n",reg_buf,bytes);
#endif
} else {
GM_DBG(PTL_GM_DBG_COMM,
"unpinning memory success,addr:%p,bytes:%d\n",reg_buf,bytes);
}
#if 0
#if 0
/*return the recv fragment to the free list */
OMPI_FREE_LIST_RETURN(&(((mca_ptl_gm_module_t
*)ptl)->gm_recv_frags_free), (ompi_list_item_t *)recv_frag);
OMPI_FREE_LIST_RETURN(
&(((mca_ptl_gm_module_t *)ptl)->gm_recv_frags_free),
(ompi_list_item_t *)recv_frag);
/* free the associated buffer */
if(recv_frag->have_allocated == true)
free(recv_frag->frag_recv.frag_base.frag_add GM_SEND_BUF_SIZE);
#endif
free(recv_frag->frag_recv.frag_base.frag_add * GM_SEND_BUF_SIZE);
#endif
}
/* XXX: will handle NACK later */
}
mca_ptl_gm_recv_frag_t* ptl_gm_data_frag( struct mca_ptl_gm_module_t *ptl,
@ -381,9 +349,9 @@ mca_ptl_gm_recv_frag_t* ptl_gm_data_frag( struct mca_ptl_gm_module_t *ptl,
mca_ptl_base_header_t *header;
header = (mca_ptl_base_header_t *)gm_ntohp(event->recv.buffer);
recv_frag = mca_ptl_gm_alloc_recv_frag( (struct mca_ptl_base_module_t*)ptl );
/* allocate a receive fragment */
recv_frag = mca_ptl_gm_alloc_recv_frag( (struct mca_ptl_base_module_t*)ptl );
recv_frag->frag_recv.frag_base.frag_owner = (struct mca_ptl_base_module_t*)ptl;
recv_frag->frag_recv.frag_base.frag_peer = NULL;
@ -395,8 +363,8 @@ mca_ptl_gm_recv_frag_t* ptl_gm_data_frag( struct mca_ptl_gm_module_t *ptl,
recv_frag->frag_progressed = 0;
recv_frag->frag_recv.frag_base.frag_header = *header;
recv_frag->frag_recv.frag_base.frag_addr = header;
/* + sizeof(mca_ptl_base_header_t);*/ /* XXX: bug */
recv_frag->frag_recv.frag_base.frag_size = gm_ntohl(event->recv.length);
recv_frag->matched = false;
@ -409,10 +377,9 @@ mca_ptl_gm_recv_frag_t* ptl_gm_data_frag( struct mca_ptl_gm_module_t *ptl,
if( matched ) {
return NULL;
}
#if DEBUG
ompi_output( 0,"matching receive not yet posted get tag %d comm %d source %d\n",
header->hdr_match.hdr_tag, header->hdr_match.hdr_contextid, header->hdr_match.hdr_src );
#endif
GM_DBG(PTL_GM_DBG_COMM,
"matching receive not yet posted get tag %d comm %d source %d\n",
header->hdr_match.hdr_tag, header->hdr_match.hdr_contextid, header->hdr_match.hdr_src );
return recv_frag;
}
@ -460,28 +427,21 @@ void mca_ptl_gm_outstanding_recv(mca_ptl_gm_module_t *ptl)
ompi_list_remove_first( (ompi_list_t *)&(ptl->gm_recv_outstanding_queue) );
printf(" the frag size to be matched is %d\n",frag->frag_recv.frag_base.frag_size);
fflush(stdout);
GM_DBG(PTL_GM_DBG_COMM," the frag size to be matched is %d\n",frag->frag_recv.frag_base.frag_size);
matched = ptl->super.ptl_match( &(ptl->super),
&(frag->frag_recv),
&(frag->frag_recv.frag_base.frag_header.hdr_match) );
printf("the value of matched is %d\n", matched);
fflush(stdout);
GM_DBG(PTL_GM_DBG_COMM,"the value of matched is %d\n", matched);
if(!matched)
{
if(!matched) {
ompi_list_append((ompi_list_t *)&(ptl->gm_recv_outstanding_queue),
(ompi_list_item_t *) frag);
}
else
{
} else {
/* if allocated buffer, free the buffer */
/* return the recv descriptor to the free list */
OMPI_FREE_LIST_RETURN(&(ptl->gm_recv_frags_free), (ompi_list_item_t *)frag);
}
}
}
@ -505,13 +465,21 @@ int mca_ptl_gm_incoming_recv (mca_ptl_gm_component_t * gm_comp)
case GM_HIGH_PEER_RECV_EVENT:
mesg = gm_ntohp(event->recv.buffer);
frag = ptl_gm_handle_recv( ptl, event );
GM_DBG(PTL_GM_DBG_COMM,"FINISHED HANDLING INCOMING EVENT\n");
if( (frag != NULL) && !(frag->matched) ) {
/* allocate temporary buffer: temporary until the fragment will be finally matched */
char* buffer = malloc( GM_SEND_BUF_SIZE );
if (NULL == buffer)
{
ompi_output(0, "[%s:%d] error in allocating memory \n",
__FILE__, __LINE__);
}
/* copy the data from the registered buffer to the newly allocated one */
memcpy( buffer, mesg, gm_ntohl(event->recv.length) );
/* associate the buffer with the unexpected fragment */
frag->frag_recv.frag_base.frag_addr = buffer;
frag->frag_recv.frag_base.frag_addr = (void *)buffer;
/* mark the fragment as having pending buffers */
frag->have_allocated_buffer = true;
@ -534,9 +502,6 @@ int mca_ptl_gm_incoming_recv (mca_ptl_gm_component_t * gm_comp)
gm_unknown(ptl->my_port, event);
}
/* process the outstanding frags in the queue */
/*mca_ptl_gm_outstanding_recv(ptl); */
}
return 0;

Просмотреть файл

@ -11,10 +11,22 @@
/* maintain list of registered buffers for send and receive */
/*struct reg_buf {*/
/*void *start; pointer to registered memory */
/*int length;*/
/*};*/
#define PTL_GM_FIRST_FRAG_SIZE (1<<14)
#define PTL_GM_DBG_NONE (0x000)
#define PTL_GM_DBG_INIT (0x001)
#define PTL_GM_DBG_COMM (0x002)
#define PTL_GM_DBG_FLAG (PTL_GM_DBG_NONE)
#define GM_DBG(flag, args...) \
do { \
if (PTL_GM_DBG_FLAG & flag) { \
char hostname[32]; gethostname(hostname, 32); \
fprintf(stderr, "[%s:%s:%d] ", \
hostname, __FUNCTION__, __LINE__); \
fprintf(stderr, args); \
} \
} while (0)
void ptl_gm_ctrl_frag(struct mca_ptl_gm_module_t *ptl,

Просмотреть файл

@ -26,5 +26,4 @@ struct mca_ptl_gm_send_request_t {
};
typedef struct mca_ptl_gm_send_request_t mca_ptl_gm_send_request_t;
#endif

Просмотреть файл

@ -16,9 +16,6 @@
#include "ptl_gm_sendfrag.h"
#include "ptl_gm_priv.h"
#define DEBUG 0
static void mca_ptl_gm_send_frag_construct (mca_ptl_gm_send_frag_t * frag);
static void mca_ptl_gm_send_frag_destruct (mca_ptl_gm_send_frag_t * frag);
@ -62,6 +59,8 @@ mca_ptl_gm_alloc_send_frag(struct mca_ptl_base_module_t *ptl,
mca_ptl_gm_send_frag_t *frag;
mca_ptl_tstamp_t tstamp = 0;
GM_DBG(PTL_GM_DBG_COMM,"INSIDE ALLOC SEND FRAG\n");
flist =&( ((mca_ptl_gm_module_t *)ptl)->gm_send_frags );
item = ompi_list_remove_first(&((flist)->super));
@ -73,6 +72,8 @@ mca_ptl_gm_alloc_send_frag(struct mca_ptl_base_module_t *ptl,
frag = (mca_ptl_gm_send_frag_t *)item;
frag->req = (struct mca_pml_base_send_request_t *)sendreq;
GM_DBG(PTL_GM_DBG_COMM," request is %p\t, frag->req = %p\n",sendreq, frag->req);
frag->type = 0 ;
return frag;
@ -101,31 +102,42 @@ int mca_ptl_gm_send_ack_init(
mca_ptl_base_header_t * hdr;
mca_pml_base_recv_request_t *request;
hdr = (mca_ptl_base_header_t *)ack->send_buf;
printf("ack buf is %p\n",ack->send_buf);
fflush(stdout);
memset(hdr, 0, sizeof(mca_ptl_base_header_t));
ack->status = -1;
ack->type = -1;
ack->wait_for_ack = 0;
ack->put_sent = -1;
ack->send_complete = -1;
GM_DBG(PTL_GM_DBG_COMM,"ack buf is %p\n",ack->send_buf);
request = frag->frag_recv.frag_request;
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_ACK;
hdr->hdr_common.hdr_flags = 0;
hdr->hdr_common.hdr_size = sizeof(mca_ptl_base_ack_header_t);
hdr->hdr_ack.hdr_src_ptr = frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_src_ptr;
hdr->hdr_ack.hdr_src_ptr.pval =
frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_src_ptr.pval;
/*assert(hdr->hdr_ack.hdr_src_ptr->req != NULL);*/
hdr->hdr_ack.hdr_dst_match.lval = 0;
hdr->hdr_ack.hdr_dst_match.pval = request; /*should this be dst_match */
hdr->hdr_ack.hdr_dst_addr.lval = 0; /*we are filling both p and val of
dest addrees */
dest address */
hdr->hdr_ack.hdr_dst_addr.pval = (void *)buffer;
hdr->hdr_ack.hdr_dst_size = size;
ack->send_frag.frag_request = 0;
ack->send_frag.frag_base.frag_peer = (struct mca_ptl_base_peer_t *)ptl_peer;
ack->send_frag.frag_base.frag_owner = (mca_ptl_base_module_t *)ptl;
ack->send_frag.frag_base.frag_addr = NULL;
ack->send_frag.frag_base.frag_size = 0;
ack->status = 1; /* was able to register memory */
ack->ptl = ptl;
ack->send_frag.frag_base.frag_header = *hdr;
ack->wait_for_ack = 0;
ack->send_frag.frag_request = 0;
ack->send_frag.frag_base.frag_peer = (struct mca_ptl_base_peer_t *)ptl_peer;
ack->send_frag.frag_base.frag_owner = (mca_ptl_base_module_t *)ptl;
ack->send_frag.frag_base.frag_addr = NULL;
ack->send_frag.frag_base.frag_size = 0;
ack->status = 1; /* was able to register memory */
ack->ptl = ptl;
ack->send_frag.frag_base.frag_header = *hdr;
ack->wait_for_ack = 0;
ack->type = ACK;
return OMPI_SUCCESS;
@ -141,8 +153,15 @@ int mca_ptl_gm_put_frag_init(
size_t* size,
int flags)
{
mca_ptl_base_header_t *hdr;
hdr = (mca_ptl_base_header_t *)putfrag->send_buf;
memset(hdr, 0, sizeof(mca_ptl_base_header_t));
putfrag->status = -1;
putfrag->type = -1;
putfrag->wait_for_ack = 0;
putfrag->put_sent = -1;
putfrag->send_complete = -1;
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FIN;
hdr->hdr_common.hdr_flags = 0;
@ -163,9 +182,9 @@ int mca_ptl_gm_put_frag_init(
putfrag->wait_for_ack = 0;
putfrag->put_sent = 0;
putfrag->type = 1;
putfrag->type = PUT;
putfrag->req = request; /* gm_send_request */
assert(putfrag->req != NULL);
return OMPI_SUCCESS;
}
@ -181,41 +200,58 @@ int mca_ptl_gm_send_frag_init(
int flags)
{
int header_length;
mca_ptl_base_header_t *hdr;
void *buffer;
buffer = sendfrag->send_buf;
hdr = (mca_ptl_base_header_t *)sendfrag->send_buf;
memset(hdr, 0, sizeof(mca_ptl_base_header_t));
sendfrag->status = -1;
sendfrag->type = -1;
sendfrag->wait_for_ack = 0;
sendfrag->put_sent = -1;
sendfrag->send_complete = -1;
hdr = (mca_ptl_base_header_t *)sendfrag->send_buf;
if (offset == 0) {
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
hdr->hdr_common.hdr_flags = flags;
hdr->hdr_common.hdr_size = sizeof(mca_ptl_base_match_header_t);
assert(sendfrag->req != NULL);
if (offset == 0)
{
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
hdr->hdr_common.hdr_flags = flags;
hdr->hdr_common.hdr_size = sizeof(mca_ptl_base_match_header_t);
hdr->hdr_frag.hdr_frag_offset = offset;
hdr->hdr_frag.hdr_frag_seq = 0;
hdr->hdr_frag.hdr_dst_ptr.lval = 0;
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag; /* pointer to the frag */
hdr->hdr_frag.hdr_frag_length = *size;
hdr->hdr_frag.hdr_frag_offset = offset;
hdr->hdr_frag.hdr_frag_seq = 0;
hdr->hdr_frag.hdr_dst_ptr.lval = 0;
hdr->hdr_frag.hdr_src_ptr.lval = 0;
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag; /* pointer to the frag */
hdr->hdr_frag.hdr_frag_length = *size;
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
hdr->hdr_match.hdr_msg_length= sendreq->req_bytes_packed;
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
header_length = sizeof (mca_ptl_base_match_header_t);
} else {
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG;
hdr->hdr_common.hdr_flags = flags;
hdr->hdr_common.hdr_size = sizeof (mca_ptl_base_frag_header_t);
hdr->hdr_frag.hdr_frag_offset = offset;
hdr->hdr_frag.hdr_frag_seq = 0;
hdr->hdr_frag.hdr_src_ptr.lval = 0;
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag;
hdr->hdr_frag.hdr_dst_ptr = sendreq->req_peer_match;
header_length = sizeof (mca_ptl_base_frag_header_t);
}
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
hdr->hdr_match.hdr_msg_length= sendreq->req_bytes_packed;
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
}
else
{
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG;
hdr->hdr_common.hdr_flags = flags;
hdr->hdr_common.hdr_size = sizeof (mca_ptl_base_frag_header_t);
hdr->hdr_frag.hdr_frag_offset = offset;
hdr->hdr_frag.hdr_frag_seq = 0;
hdr->hdr_frag.hdr_src_ptr.lval = 0;
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag;
hdr->hdr_frag.hdr_dst_ptr = sendreq->req_peer_match;
}
if (offset == 0)
sendfrag->type = MATCH;
else
sendfrag->type = FRAG;
return OMPI_SUCCESS;
}
@ -255,6 +291,7 @@ mca_ptl_gm_alloc_recv_frag( struct mca_ptl_base_module_t *ptl )
mca_ptl_gm_recv_frag_t *frag;
mca_ptl_tstamp_t tstamp = 0;
GM_DBG(PTL_GM_DBG_COMM,"INSIDE ALLOC RECV FRAG\n");
flist =&( ((mca_ptl_gm_module_t *)ptl)->gm_recv_frags_free);
item = ompi_list_remove_first(&((flist)->super));

Просмотреть файл

@ -20,6 +20,12 @@
/*#include "ptl_gm_priv.h"*/
#include "ptl_gm_peer.h"
#define MATCH 0
#define FRAG 1
#define ACK 2
#define PUT 3
OBJ_CLASS_DECLARATION (mca_ptl_gm_send_frag_t);
OBJ_CLASS_DECLARATION (mca_ptl_gm_recv_frag_t);
@ -81,13 +87,6 @@ int mca_ptl_gm_send_ack_init(
char * buffer,
int size);
/* int mca_ptl_gm_send_fini_init(
mca_ptl_gm_send_frag_t* fini,
mca_ptl_gm_module_t *ptl,
mca_ptl_gm_peer_t* ptl_peer,
mca_pml_base_send_request_t * sendreq);
*/
int
mca_ptl_gm_put_frag_init( mca_ptl_gm_send_frag_t* sendfrag,
mca_ptl_gm_peer_t * ptl_peer,

18
src/mca/ptl/gm/tests/Makefile Обычный файл
Просмотреть файл

@ -0,0 +1,18 @@
PROGS = mpi-ping
all: $(PROGS)
CC = mpicc
CFLAGS = -g
clean:
rm -f $(PROGS)
#mpi-ping-thread: mpi-ping-thread.c
# $(CC) -pthread $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS) -lpthread
mpi-ping: mpi-ping.c
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS)
run-ping: mpi-ping
mpirun -np 2 ./mpi-ping

Двоичные данные
src/mca/ptl/gm/tests/lat Исполняемый файл

Двоичный файл не отображается.

100
src/mca/ptl/gm/tests/lat.c Обычный файл
Просмотреть файл

@ -0,0 +1,100 @@
#include <stdio.h>
#include <math.h>
#include <sys/time.h>
#include "mpi.h"
#define MYBUFSIZE (1<<21)
#define CHECK 0
#define PONG 1
char s_buf[MYBUFSIZE];
char r_buf[MYBUFSIZE];
int skip = 40;
int
main (int argc, char *argv[])
{
char hostname[32];
int myid, numprocs, i, j;
double startwtime = 0.0, endwtime;
int namelen;
int size;
int loop;
MPI_Status stat;
int sleep = 1;
struct timeval t_start, t_end;
/*loop = 2;*/
if (argc < 2) {
fprintf (stderr, "Usage: %s msg_size\n", argv[0]);
return 0;
} else {
size = atoi (argv[1]);
if (argc > 2)
loop = atoi (argv[2]);
}
setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1);
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
printf("the size is %d\n",numprocs);
/* touch the data */
for (i = 0; i < size; i++) {
s_buf[i] = 'A' + i%26;
}
gethostname(hostname, 32);
fprintf(stdout, "[%s:%s:%d] done with init and barrier\n",
hostname, __FUNCTION__, __LINE__);
fflush(stdout);
for (i = 0; i < loop + skip; i++) {
if (i == skip)
gettimeofday (&t_start, 0);
if (myid == 0) {
MPI_Send (s_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD);
/*if (PONG)*/
MPI_Recv (r_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD, &stat);
} else {
MPI_Recv (r_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD, &stat);
/*if (PONG)*/
MPI_Send (s_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD);
}
if (CHECK && myid != 0) {
for (j=0; j < size; j ++) {
if (r_buf[j] != 'A' + j%26) {
fprintf(stderr, "[%s:%s] error from byte %d \n",
hostname, __FUNCTION__, j);
break;
} else {
r_buf[j] = '0';
}
}
}
}
gettimeofday (&t_end, 0);
fprintf(stdout, "[%s:%s:%d] done with pingpong\n",
hostname, __FUNCTION__, __LINE__);
fflush(stdout);
if (myid == 0) {
double latency;
latency = ((1.0e6 * t_end.tv_sec + t_end.tv_usec)
- (1.0e6 * t_start.tv_sec + t_start.tv_usec)) / (2.0 * loop);
fprintf(stdout, "length %d latency %8f\n",
size, latency);
fflush(stdout);
}
MPI_Finalize ();
return 0;
}

99
src/mca/ptl/gm/tests/lat.c.bak Обычный файл
Просмотреть файл

@ -0,0 +1,99 @@
#include <stdio.h>
#include <math.h>
#include <sys/time.h>
#include "mpi.h"
#define MYBUFSIZE (1<<21)
#define CHECK 0
#define PONG 1
char s_buf[MYBUFSIZE];
char r_buf[MYBUFSIZE];
int skip = 40;
int
main (int argc, char *argv[])
{
char hostname[32];
int myid, numprocs, i, j;
double startwtime = 0.0, endwtime;
int namelen;
int size;
int loop;
MPI_Status stat;
int sleep = 1;
struct timeval t_start, t_end;
loop = 2;
if (argc < 2) {
fprintf (stderr, "Usage: %s msg_size\n", argv[0]);
return 0;
} else {
size = atoi (argv[1]);
if (argc > 2)
loop = atoi (argv[2]);
}
setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1);
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
/* touch the data */
for (i = 0; i < size; i++) {
s_buf[i] = 'A' + i%26;
}
gethostname(hostname, 32);
fprintf(stdout, "[%s:%s:%d] done with init and barrier\n",
hostname, __FUNCTION__, __LINE__);
fflush(stdout);
for (i = 0; i < loop + skip; i++) {
if (i == skip)
gettimeofday (&t_start, 0);
if (myid == 0) {
MPI_Send (s_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD);
/*if (PONG)*/
MPI_Recv (r_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD, &stat);
} else {
MPI_Recv (r_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD, &stat);
/*if (PONG)*/
MPI_Send (s_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD);
}
if (CHECK && myid != 0) {
for (j=0; j < size; j ++) {
if (r_buf[j] != 'A' + j%26) {
fprintf(stderr, "[%s:%s] error from byte %d \n",
hostname, __FUNCTION__, j);
break;
} else {
r_buf[j] = '0';
}
}
}
}
gettimeofday (&t_end, 0);
fprintf(stdout, "[%s:%s:%d] done with pingpong\n",
hostname, __FUNCTION__, __LINE__);
fflush(stdout);
if (myid == 0) {
double latency;
latency = ((1.0e6 * t_end.tv_sec + t_end.tv_usec)
- (1.0e6 * t_start.tv_sec + t_start.tv_usec)) / (2.0 * loop);
fprintf(stdout, "length %d latency %8f\n",
size, latency);
fflush(stdout);
}
MPI_Finalize ();
return 0;
}

22
src/mca/ptl/gm/tests/lat.gm Обычный файл
Просмотреть файл

@ -0,0 +1,22 @@
length 0 latency 20.963000
length 1 latency 27.082500
length 2 latency 27.089000
length 4 latency 27.046000
length 8 latency 27.111500
length 16 latency 27.161500
length 32 latency 27.232500
length 64 latency 27.889500
length 128 latency 29.147000
length 256 latency 31.662000
length 512 latency 34.797000
length 1024 latency 42.762000
length 2048 latency 57.954500
length 4096 latency 77.715000
length 8192 latency 128.157500
length 16384 latency 306.884000
length 32768 latency 429.529000
length 65536 latency 666.954000
length 131072 latency 1126.890500
length 262144 latency 2084.019500
length 524288 latency 4060.802500
length 1048576 latency 8011.251500

22
src/mca/ptl/gm/tests/lat.gm.16k Обычный файл
Просмотреть файл

@ -0,0 +1,22 @@
length 0 latency 6.293000
length 1 latency 6.914000
length 2 latency 6.961000
length 4 latency 6.967000
length 8 latency 6.921000
length 16 latency 7.002000
length 32 latency 7.144000
length 64 latency 7.231000
length 128 latency 7.750000
length 256 latency 9.867000
length 512 latency 11.760000
length 1024 latency 14.796000
length 2048 latency 21.720000
length 4096 latency 31.352000
length 8192 latency 52.200000
length 16384 latency 213.106000
length 32768 latency 310.099000
length 65536 latency 480.808000
length 131072 latency 816.464000
length 262144 latency 1508.072000
length 524288 latency 2887.136000
length 1048576 latency 5680.983000

48
src/mca/ptl/gm/tests/lat.gm.64k Обычный файл
Просмотреть файл

@ -0,0 +1,48 @@
--------------------------------------------------
length 0 latency 6.276000
length 1 latency 6.863000
length 2 latency 6.876000
length 4 latency 6.870000
length 8 latency 6.880000
length 16 latency 6.994000
length 32 latency 7.096000
length 64 latency 7.202000
length 128 latency 7.731000
length 256 latency 9.877000
length 512 latency 11.703000
length 1024 latency 14.837000
length 2048 latency 21.597000
length 4096 latency 31.614000
length 8192 latency 52.728000
length 16384 latency 91.849000
length 32768 latency 169.010000
length 65536 latency 425.583000
length 131072 latency 778.310000
length 262144 latency 1466.517000
length 524288 latency 2840.390000
length 1048576 latency 5657.727000
---------------------------------------------------
length 0 latency 6.249000
length 1 latency 6.865000
length 2 latency 6.876000
length 4 latency 6.909000
length 8 latency 6.903000
length 16 latency 6.937000
length 32 latency 7.084000
length 64 latency 7.205000
length 128 latency 7.700000
length 256 latency 9.832000
length 512 latency 11.689000
length 1024 latency 14.779000
length 2048 latency 21.528000
length 4096 latency 31.625000
length 8192 latency 52.796000
length 16384 latency 91.786000
length 32768 latency 168.839000
length 65536 latency 428.174000
length 131072 latency 783.687000
length 262144 latency 1462.125000
length 524288 latency 2851.882000
length 1048576 latency 5655.458000

149
src/mca/ptl/gm/tests/lat.gm.dmachines Обычный файл
Просмотреть файл

@ -0,0 +1,149 @@
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 0 latency 6.276500
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 1 latency 6.908500
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 2 latency 7.014000
WARNING: session directory not removed - function not implemented
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 4 latency 6.948500
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 8 latency 6.935500
WARNING: session directory not removed - function not implemented
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 16 latency 7.053500
WARNING: session directory not removed - function not implemented
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 32 latency 7.104000
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 64 latency 7.323000
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 128 latency 7.733500
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 256 latency 9.846000
WARNING: session directory not removed - function not implemented
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 512 latency 11.741500
WARNING: session directory not removed - function not implemented
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 1024 latency 14.823500
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 2048 latency 21.604000
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 4096 latency 31.398000
WARNING: session directory not removed - function not implemented
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 8192 latency 52.105500
WARNING: session directory not removed - function not implemented
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
[d0-mellanox:main:86] done with pingpong
length 16384 latency 212.413000
[d1-mellanox:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
WARNING: session directory not removed - function not implemented
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
[d0-mellanox:main:55] done with init and barrier
[d1-mellanox:main:55] done with init and barrier
Terminated

28
src/mca/ptl/gm/tests/lat.gm.dmachines2 Обычный файл
Просмотреть файл

@ -0,0 +1,28 @@
16k Threshold 64k Threshold
(latency usecs) (latency usecs)
length 0 6.347000 6.27
length 1 6.916000 6.86
length 2 6.937000 6.87
length 4 6.948000 6.87
length 8 6.919000 6.88
length 16 6.971000 6.99
length 32 7.090000 7.09
length 64 7.230000 7.20
length 128 7.723000 7.73
length 256 9.839000 9.87
length 512 11.727000 11.70
length 1024 14.816000 14.83
length 2048 21.509000 21.59
length 4096 31.337000 31.61
length 8192 51.663000 52.72
length 16384 213.005000 91.84
length 32768 314.262000 169.01
length 65536 485.868000 425.58
length 131072 787.057000 783.62
length 262144 1464.489000 1462.16
length 524288 2871.909000 2851.17
length 1048576 5643.708000 5655.12

198
src/mca/ptl/gm/tests/lat.gm2 Обычный файл
Просмотреть файл

@ -0,0 +1,198 @@
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 0 latency 16.824000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 1 latency 20.012500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 2 latency 20.039000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a1:main:86] done with pingpong
[a0:main:86] done with pingpong
length 4 latency 20.038000
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 8 latency 20.031500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 16 latency 20.068000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 32 latency 19.955500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 64 latency 21.302000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 128 latency 22.708000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 256 latency 24.971000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 512 latency 28.205000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 1024 latency 35.137500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 2048 latency 49.086500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 4096 latency 70.039500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 8192 latency 117.148000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 16384 latency 292.963500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a1:main:86] done with pingpong
[a0:main:86] done with pingpong
length 32768 latency 419.357500
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 65536 latency 654.070000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 131072 latency 1130.282500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 262144 latency 2129.726500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 524288 latency 4156.120000
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault
/home/1/santhana/ompi_gm_install/lib/
/home/1/santhana/ompi_gm_install/lib/
[a0:main:55] done with init and barrier
[a1:main:55] done with init and barrier
[a0:main:86] done with pingpong
length 1048576 latency 8047.606500
[a1:main:86] done with pingpong
WARNING: session directory not removed - function not implemented
Segmentation fault

16
src/mca/ptl/gm/tests/lat.sh Исполняемый файл
Просмотреть файл

@ -0,0 +1,16 @@
#!/bin/bash
prog=./lat
i=$((0))
while [ $i -le $((1<<20)) ] ;
do
echo $i >&2
#prun -N 2 -B0 $prog 1000 $i
./run.sh $prog $i 500
#Allow the prun to clean up
sleep 4
if [ $i -eq 0 ] ; then
i=$((1))
else
i=$(($i*2))
fi
done

2
src/mca/ptl/gm/tests/mac Обычный файл
Просмотреть файл

@ -0,0 +1,2 @@
d0
d1

30
src/mca/ptl/gm/tests/mp-ping-results Обычный файл
Просмотреть файл

@ -0,0 +1,30 @@
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
FINISHED MPI_Comm_SIZE: size is 2
FINISHED MPI_Comm_SIZE: size is 2
mpi-ping: ping-pong (using blocking send/recv)
nprocs=2, reps=1000, min bytes=0, max bytes=1048576 inc bytes=0
0 pings 1
0 pinged 1: 0 bytes 6.28 uSec 0.00 MB/s
0 pinged 1: 1 bytes 6.93 uSec 0.14 MB/s
0 pinged 1: 2 bytes 6.93 uSec 0.29 MB/s
0 pinged 1: 4 bytes 6.96 uSec 0.57 MB/s
0 pinged 1: 8 bytes 6.94 uSec 1.15 MB/s
0 pinged 1: 16 bytes 7.01 uSec 2.28 MB/s
0 pinged 1: 32 bytes 7.12 uSec 4.49 MB/s
0 pinged 1: 64 bytes 7.26 uSec 8.82 MB/s
0 pinged 1: 128 bytes 7.74 uSec 16.54 MB/s
0 pinged 1: 256 bytes 10.01 uSec 25.57 MB/s
0 pinged 1: 512 bytes 11.82 uSec 43.32 MB/s
0 pinged 1: 1024 bytes 14.98 uSec 68.34 MB/s
0 pinged 1: 2048 bytes 21.79 uSec 93.98 MB/s
0 pinged 1: 4096 bytes 32.14 uSec 127.44 MB/s
0 pinged 1: 8192 bytes 53.68 uSec 152.60 MB/s
0 pinged 1: 16384 bytes 217.31 uSec 75.40 MB/s
0 pinged 1: 32768 bytes 319.34 uSec 102.61 MB/s
0 pinged 1: 65536 bytes 481.60 uSec 136.08 MB/s
0 pinged 1: 131072 bytes 824.24 uSec 159.02 MB/s
0 pinged 1: 262144 bytes 1506.19 uSec 174.04 MB/s
0 pinged 1: 524288 bytes 2870.90 uSec 182.62 MB/s
0 pinged 1: 1048576 bytes 5659.85 uSec 185.27 MB/s
WARNING: session directory not removed - function not implemented

Двоичные данные
src/mca/ptl/gm/tests/mpi-ping Исполняемый файл

Двоичный файл не отображается.

29
src/mca/ptl/gm/tests/mpi-ping-results Обычный файл
Просмотреть файл

@ -0,0 +1,29 @@
/home/1/santhana/openmpi_gm_install_d/lib/
/home/1/santhana/openmpi_gm_install_d/lib/
FINISHED MPI_Comm_SIZE: size is 2
mpi-ping: ping-pong (using blocking send/recv)
nprocs=2, reps=1000, min bytes=0, max bytes=1048576 inc bytes=0
0 pings 1
FINISHED MPI_Comm_SIZE: size is 2
0 pinged 1: 0 bytes 6.27 uSec 0.00 MB/s
0 pinged 1: 1 bytes 6.93 uSec 0.14 MB/s
0 pinged 1: 2 bytes 6.95 uSec 0.29 MB/s
0 pinged 1: 4 bytes 6.95 uSec 0.58 MB/s
0 pinged 1: 8 bytes 6.93 uSec 1.15 MB/s
0 pinged 1: 16 bytes 7.04 uSec 2.27 MB/s
0 pinged 1: 32 bytes 7.12 uSec 4.49 MB/s
0 pinged 1: 64 bytes 7.27 uSec 8.80 MB/s
0 pinged 1: 128 bytes 7.74 uSec 16.53 MB/s
0 pinged 1: 256 bytes 10.01 uSec 25.56 MB/s
0 pinged 1: 512 bytes 11.84 uSec 43.25 MB/s
0 pinged 1: 1024 bytes 15.02 uSec 68.19 MB/s
0 pinged 1: 2048 bytes 21.78 uSec 94.01 MB/s
0 pinged 1: 4096 bytes 32.18 uSec 127.30 MB/s
0 pinged 1: 8192 bytes 53.69 uSec 152.58 MB/s
0 pinged 1: 16384 bytes 219.26 uSec 74.72 MB/s
0 pinged 1: 32768 bytes 318.48 uSec 102.89 MB/s
0 pinged 1: 65536 bytes 481.79 uSec 136.02 MB/s
0 pinged 1: 131072 bytes 814.28 uSec 160.97 MB/s
0 pinged 1: 262144 bytes 1506.43 uSec 174.02 MB/s
0 pinged 1: 524288 bytes 2873.05 uSec 182.48 MB/s
0 pinged 1: 1048576 bytes 5651.81 uSec 185.53 MB/s

404
src/mca/ptl/gm/tests/mpi-ping.c Обычный файл
Просмотреть файл

@ -0,0 +1,404 @@
/*
* MPI ping program
*
* Patterned after the example in the Quadrics documentation
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/select.h>
#include <getopt.h>
#include "mpi.h"
static int str2size(char *str)
{
int size;
char mod[32];
switch (sscanf(str, "%d%1[mMkK]", &size, mod)) {
case 1:
return (size);
case 2:
switch (*mod) {
case 'm':
case 'M':
return (size << 20);
case 'k':
case 'K':
return (size << 10);
default:
return (size);
}
default:
return (-1);
}
}
static void usage(void)
{
fprintf(stderr,
"Usage: mpi-ping [flags] <min bytes> [<max bytes>] [<inc bytes>]\n"
" mpi-ping -h\n");
exit(EXIT_FAILURE);
}
static void help(void)
{
printf
("Usage: mpi-ping [flags] <min bytes> [<max bytes>] [<inc bytes>]\n"
"\n" " Flags may be any of\n"
" -B use blocking send/recv\n"
" -C check data\n"
" -O overlapping pings\n"
" -W perform warm-up phase\n"
" -r number repetitions to time\n"
" -h print this info\n" "\n"
" Numbers may be postfixed with 'k' or 'm'\n\n");
exit(EXIT_SUCCESS);
}
int main(int argc, char *argv[])
{
MPI_Status status;
MPI_Request recv_request;
MPI_Request send_request;
char *rbuf;
char *tbuf;
int c;
int i;
int bytes;
int nproc;
int peer;
int proc;
int r;
int tag = 0x666;
/*
* default options / arguments
*/
int reps = 10000;
int blocking = 0;
int check = 0;
int overlap = 0;
int warmup = 0;
int inc_bytes = 0;
int max_bytes = 0;
int min_bytes = 0;
setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1);
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
printf("FINISHED MPI_Comm_SIZE: size is %d\n",nproc);
fflush(stdout);
while ((c = getopt(argc, argv, "BCOWr:h")) != -1) {
switch (c) {
case 'B':
blocking = 1;
break;
case 'C':
check = 1;
break;
case 'O':
overlap = 1;
break;
case 'W':
warmup = 1;
break;
case 'r':
if ((reps = str2size(optarg)) <= 0) {
usage();
}
break;
case 'h':
help();
default:
usage();
}
}
if (optind == argc) {
min_bytes = 0;
} else if ((min_bytes = str2size(argv[optind++])) < 0) {
usage();
}
if (optind == argc) {
max_bytes = min_bytes;
} else if ((max_bytes = str2size(argv[optind++])) < min_bytes) {
usage();
}
if (optind == argc) {
inc_bytes = 0;
} else if ((inc_bytes = str2size(argv[optind++])) < 0) {
usage();
}
if (nproc == 1) {
exit(EXIT_SUCCESS);
}
if ((rbuf = (char *) malloc(max_bytes ? max_bytes : 8)) == NULL) {
perror("malloc");
exit(EXIT_FAILURE);
}
if ((tbuf = (char *) malloc(max_bytes ? max_bytes : 8)) == NULL) {
perror("malloc");
exit(EXIT_FAILURE);
}
if (check) {
for (i = 0; i < max_bytes; i++) {
tbuf[i] = i & 255;
rbuf[i] = 0;
}
}
if (proc == 0) {
if (overlap) {
printf("mpi-ping: overlapping ping-pong\n");
} else if (blocking) {
printf("mpi-ping: ping-pong (using blocking send/recv)\n");
} else {
printf("mpi-ping: ping-pong\n");
}
if (check) {
printf("data checking enabled\n");
}
printf("nprocs=%d, reps=%d, min bytes=%d, max bytes=%d inc bytes=%d\n",
nproc, reps, min_bytes, max_bytes, inc_bytes);
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
peer = proc ^ 1;
if ((peer < nproc) && (peer & 1)) {
printf("%d pings %d\n", proc, peer);
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
if (warmup) {
if (proc == 0) {
puts("warm-up phase");
fflush(stdout);
}
for (r = 0; r < reps; r++) {
if (peer >= nproc) {
break;
}
MPI_Irecv(rbuf, max_bytes, MPI_BYTE, peer, tag, MPI_COMM_WORLD,
&recv_request);
MPI_Isend(tbuf, max_bytes, MPI_BYTE, peer, tag, MPI_COMM_WORLD,
&send_request);
MPI_Wait(&send_request, &status);
MPI_Wait(&recv_request, &status);
}
if (proc == 0) {
puts("warm-up phase done");
fflush(stdout);
}
}
MPI_Barrier(MPI_COMM_WORLD);
/*
* Main loop
*/
for (bytes = min_bytes; bytes <= max_bytes;
bytes = inc_bytes ? bytes + inc_bytes : bytes ? 2 * bytes : 1) {
double t = 0.0;
double tv[2];
r = reps;
MPI_Barrier(MPI_COMM_WORLD);
if (peer < nproc) {
if (overlap) {
/*
* MPI_Isend / MPI_Irecv overlapping ping-pong
*/
tv[0] = MPI_Wtime();
for (r = 0; r < reps; r++) {
MPI_Irecv(rbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD, &recv_request);
MPI_Isend(tbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD, &send_request);
MPI_Wait(&send_request, &status);
MPI_Wait(&recv_request, &status);
if (check) {
for (i = 0; i < bytes; i++) {
if (rbuf[i] != (i & 255)) {
puts("mpi-ping: Error: Invalid data received");
}
rbuf[i] = 0;
}
}
}
tv[1] = MPI_Wtime();
} else if (blocking) {
/*
* MPI_Send / MPI_Recv ping-pong
*/
tv[0] = MPI_Wtime();
if (peer < nproc) {
if (proc & 1) {
r--;
MPI_Recv(rbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD, &status);
if (check) {
for (i = 0; i < bytes; i++) {
if (rbuf[i] != (i & 255)) {
puts("mpi-ping: Error: Invalid data received");
}
rbuf[i] = 0;
}
}
}
while (r-- > 0) {
MPI_Send(tbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD);
MPI_Recv(rbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD, &status);
if (check) {
for (i = 0; i < bytes; i++) {
if (rbuf[i] != (i & 255)) {
puts("mpi-ping: Error: Invalid data received");
}
rbuf[i] = 0;
}
}
}
if (proc & 1) {
MPI_Send(tbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD);
}
}
tv[1] = MPI_Wtime();
} else {
/*
* MPI_Isend / MPI_Irecv ping-pong
*/
tv[0] = MPI_Wtime();
if (peer < nproc) {
if (proc & 1) {
r--;
MPI_Irecv(rbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD, &recv_request);
MPI_Wait(&recv_request, &status);
if (check) {
for (i = 0; i < bytes; i++) {
if (rbuf[i] != (i & 255)) {
puts("mpi-ping: Error: Invalid data received");
}
rbuf[i] = 0;
}
}
}
while (r-- > 0) {
MPI_Isend(tbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD, &send_request);
MPI_Wait(&send_request, &status);
MPI_Irecv(rbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD, &recv_request);
MPI_Wait(&recv_request, &status);
if (check) {
for (i = 0; i < bytes; i++) {
if (rbuf[i] != (i & 255)) {
puts("mpi-ping: Error: Invalid data received");
}
rbuf[i] = 0;
}
}
}
if (proc & 1) {
MPI_Isend(tbuf, bytes, MPI_BYTE, peer, tag,
MPI_COMM_WORLD, &send_request);
MPI_Wait(&send_request, &status);
}
}
tv[1] = MPI_Wtime();
}
/*
* Calculate time interval in useconds (half round trip)
*/
t = (tv[1] - tv[0]) * 1000000.0 / (2 * reps);
}
MPI_Barrier(MPI_COMM_WORLD);
if ((peer < nproc) && (peer & 1)) {
printf("%3d pinged %3d: %8d bytes %9.2f uSec %8.2f MB/s\n",
proc, peer, bytes, t, bytes / (t));
fflush(stdout);
}
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return EXIT_SUCCESS;
}

1
src/mca/ptl/gm/tests/run.sh Исполняемый файл
Просмотреть файл

@ -0,0 +1 @@
mpirun -np 2 -hostfile mac -- ${PWD}/$1 $2 $3