Basic version of gm-ptl
This commit was SVN r2665.
Этот коммит содержится в:
родитель
5f25433bd3
Коммит
73ff3f22c5
@ -21,8 +21,6 @@
|
||||
#include "ptl_gm_peer.h"
|
||||
#include "ptl_gm_priv.h"
|
||||
|
||||
#define DEBUG 0
|
||||
|
||||
mca_ptl_gm_module_t mca_ptl_gm_module = {
|
||||
{
|
||||
&mca_ptl_gm_component.super,
|
||||
@ -40,32 +38,19 @@ mca_ptl_gm_module_t mca_ptl_gm_module = {
|
||||
mca_ptl_gm_add_procs,
|
||||
mca_ptl_gm_del_procs,
|
||||
mca_ptl_gm_finalize,
|
||||
mca_ptl_gm_send, /* JMS: Need send here */
|
||||
mca_ptl_gm_send,
|
||||
mca_ptl_gm_put,
|
||||
mca_ptl_gm_get,
|
||||
mca_ptl_gm_matched,
|
||||
mca_ptl_gm_request_init, /* JMS need request init here */
|
||||
mca_ptl_gm_request_fini, /* JMS need request fini here */
|
||||
NULL, /* JMS need match here */
|
||||
NULL, /* JMS need send_progress here */
|
||||
NULL /* JMS need recv_progress here */
|
||||
mca_ptl_gm_request_init,
|
||||
mca_ptl_gm_request_fini,
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
/*OBJ_CLASS_INSTANCE (mca_ptl_gm_recv_frag_t,*/
|
||||
/*mca_ptl_base_recv_frag_t, NULL, NULL);*/
|
||||
|
||||
OBJ_CLASS_INSTANCE (mca_ptl_gm_send_request_t,
|
||||
mca_pml_base_send_request_t, NULL, NULL);
|
||||
|
||||
|
||||
OBJ_CLASS_INSTANCE (mca_ptl_gm_peer_t, ompi_list_item_t, NULL, NULL);
|
||||
|
||||
/*
|
||||
*
|
||||
*/
|
||||
int
|
||||
mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
|
||||
size_t nprocs,
|
||||
@ -103,8 +88,6 @@ mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
|
||||
/* FIXME: */
|
||||
|
||||
for (j=0; j < num_peer_ptls; j++) {
|
||||
/*XXX: check for self */
|
||||
|
||||
ptl_peer = OBJ_NEW (mca_ptl_gm_peer_t);
|
||||
if (NULL == ptl_peer) {
|
||||
OMPI_THREAD_UNLOCK (&ptl_proc->proc_lock);
|
||||
@ -120,25 +103,20 @@ mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
|
||||
ptl_proc->proc_addrs[j].global_id,
|
||||
&lid)) {
|
||||
ompi_output (0,
|
||||
"[%s:%d] error in converting global to local id \n", __FILE__, __LINE__);
|
||||
"[%s:%d] error in converting global to local id \n",
|
||||
__FILE__, __LINE__);
|
||||
|
||||
}
|
||||
ptl_peer->local_id = lid;
|
||||
|
||||
ptl_proc->peer_arr[ptl_proc->proc_peer_count] = ptl_peer;
|
||||
ptl_proc->proc_peer_count++;
|
||||
ptl_peer->peer_addr = ptl_proc->proc_addrs + i;
|
||||
}
|
||||
ompi_bitmap_set_bit (reachable, i);
|
||||
OMPI_THREAD_UNLOCK (&ptl_proc->proc_lock);
|
||||
|
||||
peers[i] = (struct mca_ptl_base_peer_t*)ptl_peer;
|
||||
|
||||
}
|
||||
|
||||
#if DEBUG
|
||||
printf ("returning with success from gm_add_procs\n");
|
||||
#endif
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -161,7 +139,6 @@ mca_ptl_gm_del_procs (struct mca_ptl_base_module_t *ptl,
|
||||
/*
|
||||
*
|
||||
*/
|
||||
|
||||
int
|
||||
mca_ptl_gm_finalize (struct mca_ptl_base_module_t *ptl)
|
||||
{
|
||||
@ -176,6 +153,9 @@ mca_ptl_gm_request_init(struct mca_ptl_base_module_t *ptl,
|
||||
|
||||
mca_ptl_gm_send_frag_t *frag;
|
||||
struct mca_ptl_gm_send_request_t *req;
|
||||
GM_DBG(PTL_GM_DBG_COMM,
|
||||
"INSIDE REQUEST INIT: the request is %p\n",request);
|
||||
|
||||
frag = mca_ptl_gm_alloc_send_frag(ptl, request);
|
||||
|
||||
if (NULL == frag)
|
||||
@ -199,10 +179,8 @@ mca_ptl_gm_request_init(struct mca_ptl_base_module_t *ptl,
|
||||
*/
|
||||
void
|
||||
mca_ptl_gm_request_fini (struct mca_ptl_base_module_t *ptl,
|
||||
struct mca_pml_base_send_request_t *request)
|
||||
struct mca_pml_base_send_request_t *request)
|
||||
{
|
||||
|
||||
|
||||
mca_ptl_gm_send_frag_t *frag;
|
||||
|
||||
frag = ((mca_ptl_gm_send_request_t *)request)->req_frag;
|
||||
@ -224,9 +202,14 @@ mca_ptl_gm_send (struct mca_ptl_base_module_t *ptl,
|
||||
mca_ptl_gm_module_t * gm_ptl;
|
||||
int rc;
|
||||
|
||||
GM_DBG(PTL_GM_DBG_COMM,"INSIDE PTL GM SEND\n");
|
||||
|
||||
gm_ptl = (mca_ptl_gm_module_t *)ptl;
|
||||
if (offset == 0) {
|
||||
GM_DBG(PTL_GM_DBG_COMM,"OFFSET = 0\n");
|
||||
sendfrag = ((mca_ptl_gm_send_request_t *)sendreq)->req_frag;
|
||||
sendfrag->req = sendreq;
|
||||
assert(sendreq != NULL);
|
||||
} else {
|
||||
sendfrag = mca_ptl_gm_alloc_send_frag (ptl,sendreq);
|
||||
if (NULL == sendfrag) {
|
||||
@ -238,8 +221,9 @@ mca_ptl_gm_send (struct mca_ptl_base_module_t *ptl,
|
||||
|
||||
((struct mca_ptl_gm_send_request_t *)sendreq)->req_frag =sendfrag;
|
||||
((struct mca_ptl_gm_send_request_t *)sendreq)->need_ack = flags;
|
||||
rc = mca_ptl_gm_send_frag_init (sendfrag, (mca_ptl_gm_peer_t*)ptl_peer, sendreq, offset,
|
||||
&size, flags);
|
||||
rc = mca_ptl_gm_send_frag_init (sendfrag,
|
||||
(mca_ptl_gm_peer_t*)ptl_peer,
|
||||
sendreq, offset, &size, flags);
|
||||
|
||||
/*initiate the send */
|
||||
gm_ptl_peer = (mca_ptl_gm_peer_t *)ptl_peer;
|
||||
@ -248,11 +232,7 @@ mca_ptl_gm_send (struct mca_ptl_base_module_t *ptl,
|
||||
|
||||
gm_ptl->num_send_tokens--;
|
||||
/*Update offset */
|
||||
sendreq->req_offset += rc; /* XXX: should be what convertor packs */
|
||||
|
||||
/*append to the send_fragments_queue. */
|
||||
/*ompi_list_append (&(gm_ptl->gm_send_frags_queue),*/
|
||||
/*(ompi_list_item_t *) sendfrag);*/
|
||||
sendreq->req_offset += size; /* XXX: should be what convertor packs */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -280,14 +260,11 @@ mca_ptl_gm_put (struct mca_ptl_base_module_t *ptl,
|
||||
destination_buffer =(void *)( (sendreq->req_peer_addr).pval);
|
||||
|
||||
/* register the user buffer */
|
||||
if (offset > 0)
|
||||
{
|
||||
if (offset > 0) {
|
||||
status = gm_register_memory(gm_ptl->my_port, buffer_ptr, bytes_reg);
|
||||
if(GM_SUCCESS != status)
|
||||
{
|
||||
if(GM_SUCCESS != status) {
|
||||
ompi_output(0,"[%s:%d] Unable to register memory\n",__FILE__,__LINE__);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
putfrag = mca_ptl_gm_alloc_send_frag (ptl,sendreq); /*alloc_put_frag */
|
||||
@ -297,31 +274,20 @@ mca_ptl_gm_put (struct mca_ptl_base_module_t *ptl,
|
||||
((struct mca_ptl_gm_send_request_t *)sendreq)->req_frag =putfrag;
|
||||
((struct mca_ptl_gm_send_request_t *)sendreq)->need_ack = flags;
|
||||
|
||||
|
||||
|
||||
rc = mca_ptl_gm_put_frag_init(putfrag ,
|
||||
(mca_ptl_gm_peer_t*)ptl_peer,gm_ptl,
|
||||
sendreq, offset, &size, flags);
|
||||
|
||||
/* check that we have a send token available */
|
||||
rc =
|
||||
mca_ptl_gm_peer_put((mca_ptl_gm_peer_t *)ptl_peer, putfrag,
|
||||
rc = mca_ptl_gm_peer_put((mca_ptl_gm_peer_t *)ptl_peer, putfrag,
|
||||
sendreq, offset, &size, flags,
|
||||
destination_buffer, bytes_reg);
|
||||
|
||||
|
||||
|
||||
gm_ptl->num_send_tokens--;
|
||||
|
||||
sendreq->req_offset += size;
|
||||
sendreq->req_offset += size; /* should be what is returned by put */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* initiate a get.
|
||||
*/
|
||||
@ -335,15 +301,9 @@ mca_ptl_gm_get (struct mca_ptl_base_module_t *ptl,
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/* A posted receive has been matched - if required send an
|
||||
* ack back to the peer and process the fragment.
|
||||
*/
|
||||
|
||||
|
||||
void
|
||||
mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
|
||||
mca_ptl_base_recv_frag_t * frag )
|
||||
@ -353,7 +313,7 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
|
||||
mca_ptl_base_header_t *header;
|
||||
int bytes_recv, rc,rc1, total_bytes, bytes_reg;
|
||||
mca_ptl_gm_module_t *gm_ptl;
|
||||
struct iovec iov[1];
|
||||
struct iovec iov;
|
||||
mca_ptl_gm_send_frag_t *ack;
|
||||
mca_ptl_gm_recv_frag_t *recv_frag;
|
||||
char *buffer_ptr;
|
||||
@ -370,6 +330,7 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
|
||||
/* might need to send an ack back */
|
||||
recv_frag = (mca_ptl_gm_recv_frag_t *) frag;
|
||||
ack = mca_ptl_gm_alloc_send_frag(ptl,NULL);
|
||||
/*associate null request for ack*/
|
||||
|
||||
if (NULL == ack) {
|
||||
ompi_output(0,"[%s:%d] unable to alloc a gm fragment\n",
|
||||
@ -384,48 +345,43 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
|
||||
{
|
||||
|
||||
buffer_ptr = (char *)( request->req_base.req_addr );
|
||||
total_bytes = (request->req_base.req_datatype->size) *
|
||||
(request->req_base.req_count);
|
||||
|
||||
total_bytes = request->req_bytes_packed;
|
||||
/*total_bytes = (request->req_base.req_datatype->size) **/
|
||||
/*(request->req_base.req_count);*/
|
||||
bytes_recv = frag->frag_base.frag_size - header->hdr_common.hdr_size;
|
||||
bytes_reg = total_bytes - bytes_recv;
|
||||
buffer_ptr += bytes_recv;
|
||||
status = gm_register_memory(gm_ptl->my_port, buffer_ptr, bytes_reg);
|
||||
recv_frag->registered_buf = buffer_ptr;
|
||||
printf("Receiver: register addr: %p, bytes: %d\n",buffer_ptr,bytes_reg);
|
||||
fflush(stdout);
|
||||
GM_DBG(PTL_GM_DBG_COMM,"Receiver: register addr: %p, bytes: %d\n",buffer_ptr,bytes_reg);
|
||||
|
||||
if(GM_SUCCESS != status)
|
||||
{
|
||||
if(GM_SUCCESS != status) {
|
||||
ompi_output(0,"[%s:%d] Unable to register memory\n",__FILE__,__LINE__);
|
||||
}
|
||||
|
||||
/* send the regiscered memory information, send recv request * ptr */
|
||||
/* send the registered memory information, send recv request * ptr */
|
||||
rc1 = mca_ptl_gm_send_ack_init (ack, gm_ptl,
|
||||
(mca_ptl_gm_peer_t *)(recv_frag->frag_recv.frag_base.frag_peer)
|
||||
, recv_frag, buffer_ptr, bytes_reg);
|
||||
|
||||
/*XXX : put the registered memory in pin-down cache */
|
||||
|
||||
/*XXX: check this*/
|
||||
rc1 = mca_ptl_gm_peer_send ((mca_ptl_gm_peer_t *)
|
||||
(ack->send_frag.frag_base.frag_peer),
|
||||
ack,srequest,0,&size,0 );
|
||||
|
||||
mca_ptl_gm_peer_send (
|
||||
(mca_ptl_gm_peer_t *) (ack->send_frag.frag_base.frag_peer),
|
||||
ack,srequest,0,&size,0 );
|
||||
GM_DBG(PTL_GM_DBG_COMM,"RECEIVER FINISHED SENDING ACK\n");
|
||||
gm_ptl->num_send_tokens--;
|
||||
/*ompi_list_append (&(gm_ptl->gm_send_frags_queue),*/
|
||||
/*(ompi_list_item_t *) ack);*/
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
/* Here we expect that frag_addr is the begin of the buffer header included */
|
||||
iov[0].iov_base = ((char*)frag->frag_base.frag_addr) + header->hdr_common.hdr_size;
|
||||
iov.iov_base = ((char*)frag->frag_base.frag_addr) + header->hdr_common.hdr_size;
|
||||
bytes_recv = frag->frag_base.frag_size - header->hdr_common.hdr_size;
|
||||
iov[0].iov_len = bytes_recv;
|
||||
iov.iov_len = bytes_recv;
|
||||
|
||||
|
||||
if (header->hdr_frag.hdr_frag_length > 0) {
|
||||
/* ompi_proc_t *proc;
|
||||
ompi_proc_t *proc;
|
||||
|
||||
proc = ompi_comm_peer_lookup(request->req_base.req_comm,
|
||||
request->req_base.req_peer);
|
||||
@ -439,26 +395,26 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
|
||||
request->req_base.req_count,
|
||||
request->req_base.req_addr,
|
||||
header->hdr_frag.hdr_frag_offset);
|
||||
rc = ompi_convertor_unpack(&frag->frag_base.frag_convertor, &(iov[0]), 1);
|
||||
assert( rc == 1 );*/
|
||||
printf ("in matched: bytes received is %d\n", bytes_recv);
|
||||
fflush(stdout);
|
||||
memcpy(request->req_base.req_addr,iov[0].iov_base,bytes_recv);
|
||||
|
||||
rc = ompi_convertor_unpack(&frag->frag_base.frag_convertor, &(iov), 1);
|
||||
assert( rc >= 0 );
|
||||
GM_DBG(PTL_GM_DBG_COMM,"in matched: bytes received is %d\n", bytes_recv);
|
||||
/*memcpy(request->req_base.req_addr,iov.iov_base,bytes_recv);*/
|
||||
}
|
||||
|
||||
/*update progress*/
|
||||
ptl->ptl_recv_progress( ptl, request, bytes_recv, iov[0].iov_len );
|
||||
ptl->ptl_recv_progress( ptl, request, bytes_recv, iov.iov_len );
|
||||
|
||||
/* Now update the status of the fragment */
|
||||
((mca_ptl_gm_recv_frag_t*)frag)->matched = true;
|
||||
if( ((mca_ptl_gm_recv_frag_t*)frag)->have_allocated_buffer == true ){
|
||||
free( frag->frag_base.frag_addr );
|
||||
/*if( ((mca_ptl_gm_recv_frag_t*)frag)->have_allocated_buffer == true
|
||||
* ){*/
|
||||
/*free( frag->frag_base.frag_addr );*/
|
||||
((mca_ptl_gm_recv_frag_t*)frag)->have_allocated_buffer = false;
|
||||
}
|
||||
/*}*/
|
||||
/*return to free list */
|
||||
gm_ptl = (mca_ptl_gm_module_t *)ptl;
|
||||
OMPI_FREE_LIST_RETURN(&(gm_ptl->gm_recv_frags_free), (ompi_list_item_t*)frag);
|
||||
OMPI_FREE_LIST_RETURN(&(gm_ptl->gm_recv_frags_free),
|
||||
(ompi_list_item_t*)((mca_ptl_gm_recv_frag_t*)frag));
|
||||
}
|
||||
|
||||
|
||||
|
@ -19,13 +19,13 @@
|
||||
|
||||
#define MCA_PTL_GM_STATISTICS 0
|
||||
#define GM_SIZE 30
|
||||
#define THRESHOLD 16384
|
||||
#define THRESHOLD 16384
|
||||
#define MAX_GM_PORTS 16
|
||||
#define MAX_RECV_TOKENS 256
|
||||
#define PTL_GM_ADMIN_SEND_TOKENS 0
|
||||
#define PTL_GM_ADMIN_RECV_TOKENS 0
|
||||
#define GM_SEND_BUF_SIZE 16384
|
||||
#define GM_RECV_BUF_SIZE 16384
|
||||
#define GM_RECV_BUF_SIZE 16384
|
||||
#define NUM_RECV_FRAGS 100
|
||||
#define MCA_PTL_GM_FRAG_CACHED
|
||||
|
||||
|
@ -111,7 +111,8 @@ mca_ptl_gm_component_open (void)
|
||||
|
||||
/* register GM component parameters */
|
||||
mca_ptl_gm_module.super.ptl_first_frag_size =
|
||||
mca_ptl_gm_param_register_int ("first_frag_size", ((16 * 1024) - 64));
|
||||
mca_ptl_gm_param_register_int ("first_frag_size",
|
||||
((PTL_GM_FIRST_FRAG_SIZE) - 64));
|
||||
mca_ptl_gm_module.super.ptl_min_frag_size =
|
||||
mca_ptl_gm_param_register_int ("min_frag_size", 1<<16);
|
||||
mca_ptl_gm_module.super.ptl_max_frag_size =
|
||||
@ -231,7 +232,7 @@ ompi_mca_ptl_gm_init (mca_ptl_gm_component_t * gm)
|
||||
/* open the first available gm port for this board */
|
||||
board_no = i;
|
||||
for (port_no = 2; port_no < MAX_GM_PORTS; port_no++) {
|
||||
printf ("about to call open port\n");
|
||||
GM_DBG(PTL_GM_DBG_COMM,"about to call open port\n");
|
||||
if (port_no == 3) continue;
|
||||
/* port 0,1,3 reserved */
|
||||
status = gm_open (&(ptl->my_port), board_no,
|
||||
@ -338,8 +339,7 @@ ompi_mca_ptl_gm_init_sendrecv (mca_ptl_gm_component_t * gm)
|
||||
ompi_free_list_init (&(ptl->gm_recv_frags_free),
|
||||
sizeof (mca_ptl_gm_recv_frag_t),
|
||||
OBJ_CLASS (mca_ptl_gm_recv_frag_t),
|
||||
ptl->num_recv_tokens,ptl->num_recv_tokens, 1, NULL); /* not using mpool */
|
||||
|
||||
ptl->num_recv_tokens,ptl->num_recv_tokens, 1, NULL);
|
||||
|
||||
/*allocate the elements */
|
||||
free_rfragment = (mca_ptl_gm_recv_frag_t *)
|
||||
@ -435,21 +435,16 @@ mca_ptl_gm_component_control (int param, void *value, size_t size)
|
||||
/*
|
||||
* GM module progress.
|
||||
*/
|
||||
|
||||
int
|
||||
mca_ptl_gm_component_progress (mca_ptl_tstamp_t tstamp)
|
||||
{
|
||||
int rc;
|
||||
/* check the send queue to see if any pending send can proceed */
|
||||
/* check for receive and , call ptl_match to send it to the upper
|
||||
level */
|
||||
/* XXX: Do all the following inside a dispatcher, either in this routine
|
||||
* or mca_ptl_gm_incoming_recv(), YUW
|
||||
* i) check the send queue to see if any pending send can proceed
|
||||
* ii) check for recieve and , call ptl_match to send it to the upper level
|
||||
* BTW, ptl_matced is invoked inside ptl_match() via PML.
|
||||
*/
|
||||
rc = mca_ptl_gm_incoming_recv(&mca_ptl_gm_component);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
|
||||
/* check the send queue to see if any pending send can proceed */
|
||||
/* check for recieve and , call ptl_match to send it to the upper
|
||||
level */
|
||||
/* in case matched, do the appropriate queuing. */
|
||||
|
||||
}
|
||||
|
@ -23,8 +23,6 @@
|
||||
#include "ptl_gm_sendfrag.h"
|
||||
#include "ptl_gm_priv.h"
|
||||
|
||||
#define DEBUG 0
|
||||
|
||||
int mca_ptl_gm_peer_put(mca_ptl_gm_peer_t *ptl_peer,
|
||||
mca_ptl_gm_send_frag_t *fragment,
|
||||
struct mca_pml_base_send_request_t *sendreq,
|
||||
@ -58,7 +56,7 @@ int mca_ptl_gm_peer_send(mca_ptl_gm_peer_t *ptl_peer,
|
||||
size_t *size,
|
||||
int flags)
|
||||
{
|
||||
struct iovec outvec[1];
|
||||
struct iovec outvec;
|
||||
size_t size_in,size_out;
|
||||
int header_length;
|
||||
mca_ptl_base_frag_header_t* header;
|
||||
@ -68,12 +66,12 @@ int mca_ptl_gm_peer_send(mca_ptl_gm_peer_t *ptl_peer,
|
||||
|
||||
size_in = *size;
|
||||
|
||||
outvec[0].iov_base = (char*)fragment->send_buf;
|
||||
outvec.iov_base = (char*)fragment->send_buf;
|
||||
|
||||
if( (size_in + header_length) <= GM_SEND_BUF_SIZE )
|
||||
outvec[0].iov_len = size_in;
|
||||
outvec.iov_len = size_in;
|
||||
else
|
||||
outvec[0].iov_len = GM_SEND_BUF_SIZE - header_length;
|
||||
outvec.iov_len = GM_SEND_BUF_SIZE - header_length;
|
||||
|
||||
|
||||
if(size_in > 0) {
|
||||
@ -107,17 +105,18 @@ int mca_ptl_gm_peer_send(mca_ptl_gm_peer_t *ptl_peer,
|
||||
/*XXX: need to add the header */
|
||||
|
||||
/*copy the data to the registered buffer*/
|
||||
outvec[0].iov_base = ((char*)fragment->send_buf) + header_length;
|
||||
outvec.iov_base = ((char*)fragment->send_buf) + header_length;
|
||||
|
||||
if((rc = ompi_convertor_pack(convertor, &(outvec[0]), 1)) < 0)
|
||||
if((rc = ompi_convertor_pack(convertor, &(outvec), 1)) < 0)
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
/* update the fields */
|
||||
outvec[0].iov_len += header_length;
|
||||
outvec[0].iov_base = fragment->send_buf;
|
||||
outvec.iov_len += header_length;
|
||||
outvec.iov_base = fragment->send_buf;
|
||||
|
||||
/* adjust size and request offset to reflect actual number of bytes
|
||||
* packed by convertor */
|
||||
size_out = outvec[0].iov_len;
|
||||
size_out = outvec.iov_len;
|
||||
|
||||
/* initiate the gm send */
|
||||
gm_send_with_callback( ptl_peer->peer_ptl->my_port, fragment->send_buf,
|
||||
@ -126,11 +125,11 @@ int mca_ptl_gm_peer_send(mca_ptl_gm_peer_t *ptl_peer,
|
||||
|
||||
fragment->send_frag.frag_base.frag_owner = &ptl_peer->peer_ptl->super;
|
||||
fragment->send_frag.frag_base.frag_peer = (struct mca_ptl_base_peer_t*)ptl_peer;
|
||||
fragment->send_frag.frag_base.frag_addr = outvec[0].iov_base;
|
||||
fragment->send_frag.frag_base.frag_addr = outvec.iov_base;
|
||||
fragment->send_frag.frag_base.frag_size = size_out;
|
||||
|
||||
return (size_out - header_length);
|
||||
/*return OMPI_SUCCESS;*/
|
||||
*size = (size_out - header_length);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
@ -151,20 +150,13 @@ void put_callback(struct gm_port *port,void * context, gm_status_t status)
|
||||
|
||||
bytes = putfrag->send_frag.frag_base.frag_size;
|
||||
|
||||
#if DEBUG
|
||||
printf("ENTERING PUT CALLBACK\n");
|
||||
fflush(stdout);
|
||||
#endif
|
||||
GM_DBG(PTL_GM_DBG_COMM,"ENTERING PUT CALLBACK\n");
|
||||
|
||||
switch (status) {
|
||||
case GM_SUCCESS:
|
||||
/* local put completed, mark put as complete */
|
||||
|
||||
#if DEBUG
|
||||
printf("PUTCALLBACK WITH CASE GM_SUCCESS\n");
|
||||
fflush(stdout);
|
||||
#endif
|
||||
|
||||
GM_DBG(PTL_GM_DBG_COMM,"PUTCALLBACK WITH CASE GM_SUCCESS\n");
|
||||
ptl->num_send_tokens++;
|
||||
putfrag->put_sent = 1;
|
||||
|
||||
@ -172,26 +164,18 @@ void put_callback(struct gm_port *port,void * context, gm_status_t status)
|
||||
|
||||
rc = mca_ptl_gm_peer_send (putfrag->peer,putfrag,send_req,
|
||||
offset,&size,flags);
|
||||
|
||||
#if DEBUG
|
||||
printf("after issuing the put completion the request offset = %d\n",send_req->req_offset);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
assert(rc == 0);
|
||||
GM_DBG(PTL_GM_DBG_COMM,"FINISHED SENDING FIN\n");
|
||||
GM_DBG(PTL_GM_DBG_COMM,"after issuing the put completion the request offset = %d\n",send_req->req_offset);
|
||||
|
||||
/* deregister the user memory */
|
||||
status = gm_deregister_memory(ptl->my_port, (char *)(putfrag->registered_buf), bytes);
|
||||
|
||||
if(GM_SUCCESS != status)
|
||||
{
|
||||
#if DEBUG
|
||||
ompi_output(0," unpinning memory failed\n");
|
||||
#endif
|
||||
}
|
||||
else
|
||||
{
|
||||
#if DEBUG
|
||||
ompi_output(0," unpinning %d bytes of memory success\n",bytes);
|
||||
#endif
|
||||
if(GM_SUCCESS != status) {
|
||||
ompi_output(0," unpinning memory failed\n");
|
||||
}
|
||||
else {
|
||||
GM_DBG(PTL_GM_DBG_COMM, " unpinning %d bytes of memory success\n",bytes);
|
||||
}
|
||||
break;
|
||||
|
||||
@ -228,7 +212,7 @@ void send_callback(struct gm_port *port,void * context, gm_status_t status)
|
||||
header = (mca_ptl_base_header_t*)frag->send_buf;
|
||||
header_length = ((mca_ptl_base_header_t*)header)->hdr_common.hdr_size;
|
||||
|
||||
if (frag->type == 1)
|
||||
if (frag->type == PUT)
|
||||
{
|
||||
bytes = header->hdr_ack.hdr_dst_size;
|
||||
}
|
||||
@ -242,34 +226,23 @@ void send_callback(struct gm_port *port,void * context, gm_status_t status)
|
||||
frag->wait_for_ack = 1;
|
||||
}
|
||||
|
||||
|
||||
switch (status) {
|
||||
case GM_SUCCESS:
|
||||
|
||||
/* send completed, can reuse the user buffer */
|
||||
#if DEBUG
|
||||
printf("SENDCALLBACK WITH CASE GM_SUCCESS\n");
|
||||
fflush(stdout);
|
||||
#endif
|
||||
GM_DBG(PTL_GM_DBG_COMM,
|
||||
"SENDCALLBACK WITH CASE GM_SUCCESS: frag->wait_for_ack = %d\n",
|
||||
frag->wait_for_ack);
|
||||
|
||||
ptl->num_send_tokens++;
|
||||
frag->send_complete = 1;
|
||||
|
||||
if (frag->wait_for_ack == 0 && (gm_send_req != NULL))
|
||||
{
|
||||
#if DEBUG
|
||||
printf("inside send callback : calling send progress bytes = %d\n",bytes);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
ptl->super.ptl_send_progress( (mca_ptl_base_module_t*)ptl,
|
||||
gm_send_req, bytes );
|
||||
|
||||
if ((frag->wait_for_ack == 0) && (gm_send_req != NULL)) {
|
||||
GM_DBG(PTL_GM_DBG_COMM,"inside send callback : calling send progress bytes = %d\n",bytes);
|
||||
ptl->super.ptl_send_progress( (mca_ptl_base_module_t*)ptl,
|
||||
gm_send_req, bytes );
|
||||
}
|
||||
|
||||
OMPI_FREE_LIST_RETURN(&(ptl->gm_send_frags), (ompi_list_item_t *)frag);
|
||||
/*list = (ompi_list_t *)(&(ptl->gm_send_frags_queue));*/
|
||||
/*ompi_list_remove_first(list);*/
|
||||
|
||||
break;
|
||||
|
||||
case GM_SEND_TIMED_OUT:
|
||||
@ -285,6 +258,8 @@ void send_callback(struct gm_port *port,void * context, gm_status_t status)
|
||||
"[%s:%d] error in message completion\n",__FILE__,__LINE__);
|
||||
break;
|
||||
}
|
||||
|
||||
GM_DBG(PTL_GM_DBG_COMM,"RETURNING FROM SEND_CALLBACK\n");
|
||||
}
|
||||
|
||||
|
||||
@ -302,6 +277,7 @@ void ptl_gm_ctrl_frag(struct mca_ptl_gm_module_t *ptl,
|
||||
{
|
||||
frag = (mca_ptl_gm_send_frag_t *)header->hdr_ack.hdr_src_ptr.pval;
|
||||
req = (mca_pml_base_send_request_t *) frag->req;
|
||||
assert(req != NULL);
|
||||
req->req_peer_match.pval = header->hdr_ack.hdr_dst_match.pval;
|
||||
req->req_peer_addr.pval = header->hdr_ack.hdr_dst_addr.pval;
|
||||
req->req_peer_size = header->hdr_ack.hdr_dst_size;
|
||||
@ -325,14 +301,13 @@ void ptl_gm_ctrl_frag(struct mca_ptl_gm_module_t *ptl,
|
||||
if(header->hdr_common.hdr_type == MCA_PTL_HDR_TYPE_FIN)
|
||||
{
|
||||
|
||||
GM_DBG(PTL_GM_DBG_COMM,"CASE: HDR_TYPE_FIN\n");
|
||||
|
||||
request = (mca_pml_base_recv_request_t*)
|
||||
header->hdr_ack.hdr_dst_match.pval;
|
||||
/* call receive progress and indicate the recv has been completed */
|
||||
|
||||
#if DEBUG
|
||||
printf("Calling recv_progress with bytes = %d\n",header->hdr_ack.hdr_dst_size);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
GM_DBG(PTL_GM_DBG_COMM,"Calling recv_progress with bytes = %d\n",header->hdr_ack.hdr_dst_size);
|
||||
ptl->super.ptl_recv_progress (
|
||||
(mca_ptl_base_module_t *) ptl,
|
||||
request ,
|
||||
@ -344,33 +319,26 @@ void ptl_gm_ctrl_frag(struct mca_ptl_gm_module_t *ptl,
|
||||
status = gm_deregister_memory(ptl->my_port, reg_buf,
|
||||
bytes);
|
||||
|
||||
if(GM_SUCCESS != status)
|
||||
{
|
||||
#if DEBUG
|
||||
if(GM_SUCCESS != status) {
|
||||
ompi_output(0," unpinning memory failed\n");
|
||||
#endif
|
||||
}
|
||||
else
|
||||
{
|
||||
#if DEBUG
|
||||
ompi_output(0,"unpinning memory success,addr:%p,bytes:%d\n",reg_buf,bytes);
|
||||
#endif
|
||||
} else {
|
||||
GM_DBG(PTL_GM_DBG_COMM,
|
||||
"unpinning memory success,addr:%p,bytes:%d\n",reg_buf,bytes);
|
||||
}
|
||||
|
||||
#if 0
|
||||
#if 0
|
||||
/*return the recv fragment to the free list */
|
||||
OMPI_FREE_LIST_RETURN(&(((mca_ptl_gm_module_t
|
||||
*)ptl)->gm_recv_frags_free), (ompi_list_item_t *)recv_frag);
|
||||
OMPI_FREE_LIST_RETURN(
|
||||
&(((mca_ptl_gm_module_t *)ptl)->gm_recv_frags_free),
|
||||
(ompi_list_item_t *)recv_frag);
|
||||
|
||||
/* free the associated buffer */
|
||||
if(recv_frag->have_allocated == true)
|
||||
free(recv_frag->frag_recv.frag_base.frag_add GM_SEND_BUF_SIZE);
|
||||
#endif
|
||||
|
||||
free(recv_frag->frag_recv.frag_base.frag_add * GM_SEND_BUF_SIZE);
|
||||
#endif
|
||||
}
|
||||
|
||||
/* XXX: will handle NACK later */
|
||||
|
||||
}
|
||||
|
||||
mca_ptl_gm_recv_frag_t* ptl_gm_data_frag( struct mca_ptl_gm_module_t *ptl,
|
||||
@ -381,9 +349,9 @@ mca_ptl_gm_recv_frag_t* ptl_gm_data_frag( struct mca_ptl_gm_module_t *ptl,
|
||||
mca_ptl_base_header_t *header;
|
||||
|
||||
header = (mca_ptl_base_header_t *)gm_ntohp(event->recv.buffer);
|
||||
|
||||
recv_frag = mca_ptl_gm_alloc_recv_frag( (struct mca_ptl_base_module_t*)ptl );
|
||||
|
||||
/* allocate a receive fragment */
|
||||
recv_frag = mca_ptl_gm_alloc_recv_frag( (struct mca_ptl_base_module_t*)ptl );
|
||||
|
||||
recv_frag->frag_recv.frag_base.frag_owner = (struct mca_ptl_base_module_t*)ptl;
|
||||
recv_frag->frag_recv.frag_base.frag_peer = NULL;
|
||||
@ -395,8 +363,8 @@ mca_ptl_gm_recv_frag_t* ptl_gm_data_frag( struct mca_ptl_gm_module_t *ptl,
|
||||
recv_frag->frag_progressed = 0;
|
||||
|
||||
recv_frag->frag_recv.frag_base.frag_header = *header;
|
||||
|
||||
recv_frag->frag_recv.frag_base.frag_addr = header;
|
||||
/* + sizeof(mca_ptl_base_header_t);*/ /* XXX: bug */
|
||||
recv_frag->frag_recv.frag_base.frag_size = gm_ntohl(event->recv.length);
|
||||
|
||||
recv_frag->matched = false;
|
||||
@ -409,10 +377,9 @@ mca_ptl_gm_recv_frag_t* ptl_gm_data_frag( struct mca_ptl_gm_module_t *ptl,
|
||||
if( matched ) {
|
||||
return NULL;
|
||||
}
|
||||
#if DEBUG
|
||||
ompi_output( 0,"matching receive not yet posted get tag %d comm %d source %d\n",
|
||||
header->hdr_match.hdr_tag, header->hdr_match.hdr_contextid, header->hdr_match.hdr_src );
|
||||
#endif
|
||||
GM_DBG(PTL_GM_DBG_COMM,
|
||||
"matching receive not yet posted get tag %d comm %d source %d\n",
|
||||
header->hdr_match.hdr_tag, header->hdr_match.hdr_contextid, header->hdr_match.hdr_src );
|
||||
return recv_frag;
|
||||
}
|
||||
|
||||
@ -460,28 +427,21 @@ void mca_ptl_gm_outstanding_recv(mca_ptl_gm_module_t *ptl)
|
||||
ompi_list_remove_first( (ompi_list_t *)&(ptl->gm_recv_outstanding_queue) );
|
||||
|
||||
|
||||
printf(" the frag size to be matched is %d\n",frag->frag_recv.frag_base.frag_size);
|
||||
fflush(stdout);
|
||||
GM_DBG(PTL_GM_DBG_COMM," the frag size to be matched is %d\n",frag->frag_recv.frag_base.frag_size);
|
||||
matched = ptl->super.ptl_match( &(ptl->super),
|
||||
&(frag->frag_recv),
|
||||
&(frag->frag_recv.frag_base.frag_header.hdr_match) );
|
||||
|
||||
printf("the value of matched is %d\n", matched);
|
||||
fflush(stdout);
|
||||
GM_DBG(PTL_GM_DBG_COMM,"the value of matched is %d\n", matched);
|
||||
|
||||
if(!matched)
|
||||
{
|
||||
if(!matched) {
|
||||
ompi_list_append((ompi_list_t *)&(ptl->gm_recv_outstanding_queue),
|
||||
(ompi_list_item_t *) frag);
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
/* if allocated buffer, free the buffer */
|
||||
|
||||
/* return the recv descriptor to the free list */
|
||||
OMPI_FREE_LIST_RETURN(&(ptl->gm_recv_frags_free), (ompi_list_item_t *)frag);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
@ -505,13 +465,21 @@ int mca_ptl_gm_incoming_recv (mca_ptl_gm_component_t * gm_comp)
|
||||
case GM_HIGH_PEER_RECV_EVENT:
|
||||
mesg = gm_ntohp(event->recv.buffer);
|
||||
frag = ptl_gm_handle_recv( ptl, event );
|
||||
GM_DBG(PTL_GM_DBG_COMM,"FINISHED HANDLING INCOMING EVENT\n");
|
||||
|
||||
if( (frag != NULL) && !(frag->matched) ) {
|
||||
/* allocate temporary buffer: temporary until the fragment will be finally matched */
|
||||
char* buffer = malloc( GM_SEND_BUF_SIZE );
|
||||
if (NULL == buffer)
|
||||
{
|
||||
ompi_output(0, "[%s:%d] error in allocating memory \n",
|
||||
__FILE__, __LINE__);
|
||||
|
||||
}
|
||||
/* copy the data from the registered buffer to the newly allocated one */
|
||||
memcpy( buffer, mesg, gm_ntohl(event->recv.length) );
|
||||
/* associate the buffer with the unexpected fragment */
|
||||
frag->frag_recv.frag_base.frag_addr = buffer;
|
||||
frag->frag_recv.frag_base.frag_addr = (void *)buffer;
|
||||
/* mark the fragment as having pending buffers */
|
||||
frag->have_allocated_buffer = true;
|
||||
|
||||
@ -534,9 +502,6 @@ int mca_ptl_gm_incoming_recv (mca_ptl_gm_component_t * gm_comp)
|
||||
gm_unknown(ptl->my_port, event);
|
||||
|
||||
}
|
||||
/* process the outstanding frags in the queue */
|
||||
/*mca_ptl_gm_outstanding_recv(ptl); */
|
||||
|
||||
|
||||
}
|
||||
return 0;
|
||||
|
@ -11,10 +11,22 @@
|
||||
|
||||
/* maintain list of registered buffers for send and receive */
|
||||
|
||||
/*struct reg_buf {*/
|
||||
/*void *start; pointer to registered memory */
|
||||
/*int length;*/
|
||||
/*};*/
|
||||
#define PTL_GM_FIRST_FRAG_SIZE (1<<14)
|
||||
#define PTL_GM_DBG_NONE (0x000)
|
||||
#define PTL_GM_DBG_INIT (0x001)
|
||||
#define PTL_GM_DBG_COMM (0x002)
|
||||
|
||||
#define PTL_GM_DBG_FLAG (PTL_GM_DBG_NONE)
|
||||
|
||||
#define GM_DBG(flag, args...) \
|
||||
do { \
|
||||
if (PTL_GM_DBG_FLAG & flag) { \
|
||||
char hostname[32]; gethostname(hostname, 32); \
|
||||
fprintf(stderr, "[%s:%s:%d] ", \
|
||||
hostname, __FUNCTION__, __LINE__); \
|
||||
fprintf(stderr, args); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
|
||||
void ptl_gm_ctrl_frag(struct mca_ptl_gm_module_t *ptl,
|
||||
|
@ -26,5 +26,4 @@ struct mca_ptl_gm_send_request_t {
|
||||
};
|
||||
typedef struct mca_ptl_gm_send_request_t mca_ptl_gm_send_request_t;
|
||||
|
||||
|
||||
#endif
|
||||
|
@ -16,9 +16,6 @@
|
||||
#include "ptl_gm_sendfrag.h"
|
||||
#include "ptl_gm_priv.h"
|
||||
|
||||
#define DEBUG 0
|
||||
|
||||
|
||||
static void mca_ptl_gm_send_frag_construct (mca_ptl_gm_send_frag_t * frag);
|
||||
static void mca_ptl_gm_send_frag_destruct (mca_ptl_gm_send_frag_t * frag);
|
||||
|
||||
@ -62,6 +59,8 @@ mca_ptl_gm_alloc_send_frag(struct mca_ptl_base_module_t *ptl,
|
||||
mca_ptl_gm_send_frag_t *frag;
|
||||
mca_ptl_tstamp_t tstamp = 0;
|
||||
|
||||
GM_DBG(PTL_GM_DBG_COMM,"INSIDE ALLOC SEND FRAG\n");
|
||||
|
||||
flist =&( ((mca_ptl_gm_module_t *)ptl)->gm_send_frags );
|
||||
item = ompi_list_remove_first(&((flist)->super));
|
||||
|
||||
@ -73,6 +72,8 @@ mca_ptl_gm_alloc_send_frag(struct mca_ptl_base_module_t *ptl,
|
||||
|
||||
frag = (mca_ptl_gm_send_frag_t *)item;
|
||||
frag->req = (struct mca_pml_base_send_request_t *)sendreq;
|
||||
GM_DBG(PTL_GM_DBG_COMM," request is %p\t, frag->req = %p\n",sendreq, frag->req);
|
||||
|
||||
frag->type = 0 ;
|
||||
return frag;
|
||||
|
||||
@ -101,31 +102,42 @@ int mca_ptl_gm_send_ack_init(
|
||||
mca_ptl_base_header_t * hdr;
|
||||
mca_pml_base_recv_request_t *request;
|
||||
hdr = (mca_ptl_base_header_t *)ack->send_buf;
|
||||
printf("ack buf is %p\n",ack->send_buf);
|
||||
fflush(stdout);
|
||||
memset(hdr, 0, sizeof(mca_ptl_base_header_t));
|
||||
ack->status = -1;
|
||||
ack->type = -1;
|
||||
ack->wait_for_ack = 0;
|
||||
ack->put_sent = -1;
|
||||
ack->send_complete = -1;
|
||||
|
||||
|
||||
GM_DBG(PTL_GM_DBG_COMM,"ack buf is %p\n",ack->send_buf);
|
||||
|
||||
request = frag->frag_recv.frag_request;
|
||||
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_ACK;
|
||||
hdr->hdr_common.hdr_flags = 0;
|
||||
hdr->hdr_common.hdr_size = sizeof(mca_ptl_base_ack_header_t);
|
||||
|
||||
hdr->hdr_ack.hdr_src_ptr = frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_src_ptr;
|
||||
hdr->hdr_ack.hdr_src_ptr.pval =
|
||||
frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_src_ptr.pval;
|
||||
/*assert(hdr->hdr_ack.hdr_src_ptr->req != NULL);*/
|
||||
hdr->hdr_ack.hdr_dst_match.lval = 0;
|
||||
hdr->hdr_ack.hdr_dst_match.pval = request; /*should this be dst_match */
|
||||
hdr->hdr_ack.hdr_dst_addr.lval = 0; /*we are filling both p and val of
|
||||
dest addrees */
|
||||
dest address */
|
||||
hdr->hdr_ack.hdr_dst_addr.pval = (void *)buffer;
|
||||
hdr->hdr_ack.hdr_dst_size = size;
|
||||
|
||||
ack->send_frag.frag_request = 0;
|
||||
ack->send_frag.frag_base.frag_peer = (struct mca_ptl_base_peer_t *)ptl_peer;
|
||||
ack->send_frag.frag_base.frag_owner = (mca_ptl_base_module_t *)ptl;
|
||||
ack->send_frag.frag_base.frag_addr = NULL;
|
||||
ack->send_frag.frag_base.frag_size = 0;
|
||||
ack->status = 1; /* was able to register memory */
|
||||
ack->ptl = ptl;
|
||||
ack->send_frag.frag_base.frag_header = *hdr;
|
||||
ack->wait_for_ack = 0;
|
||||
|
||||
ack->send_frag.frag_request = 0;
|
||||
ack->send_frag.frag_base.frag_peer = (struct mca_ptl_base_peer_t *)ptl_peer;
|
||||
ack->send_frag.frag_base.frag_owner = (mca_ptl_base_module_t *)ptl;
|
||||
ack->send_frag.frag_base.frag_addr = NULL;
|
||||
ack->send_frag.frag_base.frag_size = 0;
|
||||
ack->status = 1; /* was able to register memory */
|
||||
ack->ptl = ptl;
|
||||
ack->send_frag.frag_base.frag_header = *hdr;
|
||||
ack->wait_for_ack = 0;
|
||||
ack->type = ACK;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
|
||||
@ -141,8 +153,15 @@ int mca_ptl_gm_put_frag_init(
|
||||
size_t* size,
|
||||
int flags)
|
||||
{
|
||||
|
||||
mca_ptl_base_header_t *hdr;
|
||||
hdr = (mca_ptl_base_header_t *)putfrag->send_buf;
|
||||
memset(hdr, 0, sizeof(mca_ptl_base_header_t));
|
||||
putfrag->status = -1;
|
||||
putfrag->type = -1;
|
||||
putfrag->wait_for_ack = 0;
|
||||
putfrag->put_sent = -1;
|
||||
putfrag->send_complete = -1;
|
||||
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FIN;
|
||||
hdr->hdr_common.hdr_flags = 0;
|
||||
@ -163,9 +182,9 @@ int mca_ptl_gm_put_frag_init(
|
||||
|
||||
putfrag->wait_for_ack = 0;
|
||||
putfrag->put_sent = 0;
|
||||
putfrag->type = 1;
|
||||
putfrag->type = PUT;
|
||||
putfrag->req = request; /* gm_send_request */
|
||||
|
||||
assert(putfrag->req != NULL);
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
@ -181,41 +200,58 @@ int mca_ptl_gm_send_frag_init(
|
||||
int flags)
|
||||
|
||||
{
|
||||
int header_length;
|
||||
mca_ptl_base_header_t *hdr;
|
||||
void *buffer;
|
||||
|
||||
buffer = sendfrag->send_buf;
|
||||
hdr = (mca_ptl_base_header_t *)sendfrag->send_buf;
|
||||
memset(hdr, 0, sizeof(mca_ptl_base_header_t));
|
||||
|
||||
sendfrag->status = -1;
|
||||
sendfrag->type = -1;
|
||||
sendfrag->wait_for_ack = 0;
|
||||
sendfrag->put_sent = -1;
|
||||
sendfrag->send_complete = -1;
|
||||
|
||||
hdr = (mca_ptl_base_header_t *)sendfrag->send_buf;
|
||||
if (offset == 0) {
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_common.hdr_size = sizeof(mca_ptl_base_match_header_t);
|
||||
assert(sendfrag->req != NULL);
|
||||
if (offset == 0)
|
||||
{
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_common.hdr_size = sizeof(mca_ptl_base_match_header_t);
|
||||
|
||||
hdr->hdr_frag.hdr_frag_offset = offset;
|
||||
hdr->hdr_frag.hdr_frag_seq = 0;
|
||||
hdr->hdr_frag.hdr_dst_ptr.lval = 0;
|
||||
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag; /* pointer to the frag */
|
||||
hdr->hdr_frag.hdr_frag_length = *size;
|
||||
hdr->hdr_frag.hdr_frag_offset = offset;
|
||||
hdr->hdr_frag.hdr_frag_seq = 0;
|
||||
hdr->hdr_frag.hdr_dst_ptr.lval = 0;
|
||||
hdr->hdr_frag.hdr_src_ptr.lval = 0;
|
||||
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag; /* pointer to the frag */
|
||||
|
||||
hdr->hdr_frag.hdr_frag_length = *size;
|
||||
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length= sendreq->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
|
||||
header_length = sizeof (mca_ptl_base_match_header_t);
|
||||
} else {
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG;
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_common.hdr_size = sizeof (mca_ptl_base_frag_header_t);
|
||||
hdr->hdr_frag.hdr_frag_offset = offset;
|
||||
hdr->hdr_frag.hdr_frag_seq = 0;
|
||||
hdr->hdr_frag.hdr_src_ptr.lval = 0;
|
||||
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag;
|
||||
hdr->hdr_frag.hdr_dst_ptr = sendreq->req_peer_match;
|
||||
header_length = sizeof (mca_ptl_base_frag_header_t);
|
||||
}
|
||||
hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
|
||||
hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
|
||||
hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
|
||||
hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
|
||||
hdr->hdr_match.hdr_msg_length= sendreq->req_bytes_packed;
|
||||
hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
|
||||
}
|
||||
else
|
||||
{
|
||||
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_FRAG;
|
||||
hdr->hdr_common.hdr_flags = flags;
|
||||
hdr->hdr_common.hdr_size = sizeof (mca_ptl_base_frag_header_t);
|
||||
|
||||
hdr->hdr_frag.hdr_frag_offset = offset;
|
||||
hdr->hdr_frag.hdr_frag_seq = 0;
|
||||
hdr->hdr_frag.hdr_src_ptr.lval = 0;
|
||||
hdr->hdr_frag.hdr_src_ptr.pval = sendfrag;
|
||||
hdr->hdr_frag.hdr_dst_ptr = sendreq->req_peer_match;
|
||||
}
|
||||
|
||||
if (offset == 0)
|
||||
sendfrag->type = MATCH;
|
||||
else
|
||||
sendfrag->type = FRAG;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
@ -255,6 +291,7 @@ mca_ptl_gm_alloc_recv_frag( struct mca_ptl_base_module_t *ptl )
|
||||
mca_ptl_gm_recv_frag_t *frag;
|
||||
mca_ptl_tstamp_t tstamp = 0;
|
||||
|
||||
GM_DBG(PTL_GM_DBG_COMM,"INSIDE ALLOC RECV FRAG\n");
|
||||
flist =&( ((mca_ptl_gm_module_t *)ptl)->gm_recv_frags_free);
|
||||
item = ompi_list_remove_first(&((flist)->super));
|
||||
|
||||
|
@ -20,6 +20,12 @@
|
||||
/*#include "ptl_gm_priv.h"*/
|
||||
#include "ptl_gm_peer.h"
|
||||
|
||||
|
||||
#define MATCH 0
|
||||
#define FRAG 1
|
||||
#define ACK 2
|
||||
#define PUT 3
|
||||
|
||||
OBJ_CLASS_DECLARATION (mca_ptl_gm_send_frag_t);
|
||||
OBJ_CLASS_DECLARATION (mca_ptl_gm_recv_frag_t);
|
||||
|
||||
@ -81,13 +87,6 @@ int mca_ptl_gm_send_ack_init(
|
||||
char * buffer,
|
||||
int size);
|
||||
|
||||
/* int mca_ptl_gm_send_fini_init(
|
||||
mca_ptl_gm_send_frag_t* fini,
|
||||
mca_ptl_gm_module_t *ptl,
|
||||
mca_ptl_gm_peer_t* ptl_peer,
|
||||
mca_pml_base_send_request_t * sendreq);
|
||||
*/
|
||||
|
||||
int
|
||||
mca_ptl_gm_put_frag_init( mca_ptl_gm_send_frag_t* sendfrag,
|
||||
mca_ptl_gm_peer_t * ptl_peer,
|
||||
|
18
src/mca/ptl/gm/tests/Makefile
Обычный файл
18
src/mca/ptl/gm/tests/Makefile
Обычный файл
@ -0,0 +1,18 @@
|
||||
PROGS = mpi-ping
|
||||
|
||||
all: $(PROGS)
|
||||
|
||||
CC = mpicc
|
||||
CFLAGS = -g
|
||||
|
||||
clean:
|
||||
rm -f $(PROGS)
|
||||
|
||||
#mpi-ping-thread: mpi-ping-thread.c
|
||||
# $(CC) -pthread $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS) -lpthread
|
||||
|
||||
mpi-ping: mpi-ping.c
|
||||
$(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS)
|
||||
|
||||
run-ping: mpi-ping
|
||||
mpirun -np 2 ./mpi-ping
|
Двоичные данные
src/mca/ptl/gm/tests/lat
Исполняемый файл
Двоичные данные
src/mca/ptl/gm/tests/lat
Исполняемый файл
Двоичный файл не отображается.
100
src/mca/ptl/gm/tests/lat.c
Обычный файл
100
src/mca/ptl/gm/tests/lat.c
Обычный файл
@ -0,0 +1,100 @@
|
||||
|
||||
#include <stdio.h>
|
||||
#include <math.h>
|
||||
#include <sys/time.h>
|
||||
#include "mpi.h"
|
||||
|
||||
#define MYBUFSIZE (1<<21)
|
||||
#define CHECK 0
|
||||
#define PONG 1
|
||||
|
||||
char s_buf[MYBUFSIZE];
|
||||
char r_buf[MYBUFSIZE];
|
||||
int skip = 40;
|
||||
|
||||
int
|
||||
main (int argc, char *argv[])
|
||||
{
|
||||
char hostname[32];
|
||||
|
||||
int myid, numprocs, i, j;
|
||||
double startwtime = 0.0, endwtime;
|
||||
int namelen;
|
||||
int size;
|
||||
int loop;
|
||||
MPI_Status stat;
|
||||
int sleep = 1;
|
||||
|
||||
struct timeval t_start, t_end;
|
||||
|
||||
/*loop = 2;*/
|
||||
|
||||
if (argc < 2) {
|
||||
fprintf (stderr, "Usage: %s msg_size\n", argv[0]);
|
||||
return 0;
|
||||
} else {
|
||||
size = atoi (argv[1]);
|
||||
if (argc > 2)
|
||||
loop = atoi (argv[2]);
|
||||
}
|
||||
|
||||
setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1);
|
||||
|
||||
MPI_Init (&argc, &argv);
|
||||
MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
|
||||
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
|
||||
printf("the size is %d\n",numprocs);
|
||||
|
||||
/* touch the data */
|
||||
for (i = 0; i < size; i++) {
|
||||
s_buf[i] = 'A' + i%26;
|
||||
}
|
||||
|
||||
gethostname(hostname, 32);
|
||||
|
||||
fprintf(stdout, "[%s:%s:%d] done with init and barrier\n",
|
||||
hostname, __FUNCTION__, __LINE__);
|
||||
fflush(stdout);
|
||||
|
||||
for (i = 0; i < loop + skip; i++) {
|
||||
if (i == skip)
|
||||
gettimeofday (&t_start, 0);
|
||||
if (myid == 0) {
|
||||
MPI_Send (s_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD);
|
||||
/*if (PONG)*/
|
||||
MPI_Recv (r_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD, &stat);
|
||||
} else {
|
||||
MPI_Recv (r_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD, &stat);
|
||||
/*if (PONG)*/
|
||||
MPI_Send (s_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD);
|
||||
}
|
||||
|
||||
if (CHECK && myid != 0) {
|
||||
for (j=0; j < size; j ++) {
|
||||
if (r_buf[j] != 'A' + j%26) {
|
||||
fprintf(stderr, "[%s:%s] error from byte %d \n",
|
||||
hostname, __FUNCTION__, j);
|
||||
break;
|
||||
} else {
|
||||
r_buf[j] = '0';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
gettimeofday (&t_end, 0);
|
||||
|
||||
fprintf(stdout, "[%s:%s:%d] done with pingpong\n",
|
||||
hostname, __FUNCTION__, __LINE__);
|
||||
fflush(stdout);
|
||||
|
||||
if (myid == 0) {
|
||||
double latency;
|
||||
latency = ((1.0e6 * t_end.tv_sec + t_end.tv_usec)
|
||||
- (1.0e6 * t_start.tv_sec + t_start.tv_usec)) / (2.0 * loop);
|
||||
fprintf(stdout, "length %d latency %8f\n",
|
||||
size, latency);
|
||||
fflush(stdout);
|
||||
}
|
||||
MPI_Finalize ();
|
||||
return 0;
|
||||
}
|
99
src/mca/ptl/gm/tests/lat.c.bak
Обычный файл
99
src/mca/ptl/gm/tests/lat.c.bak
Обычный файл
@ -0,0 +1,99 @@
|
||||
|
||||
#include <stdio.h>
|
||||
#include <math.h>
|
||||
#include <sys/time.h>
|
||||
#include "mpi.h"
|
||||
|
||||
#define MYBUFSIZE (1<<21)
|
||||
#define CHECK 0
|
||||
#define PONG 1
|
||||
|
||||
char s_buf[MYBUFSIZE];
|
||||
char r_buf[MYBUFSIZE];
|
||||
int skip = 40;
|
||||
|
||||
int
|
||||
main (int argc, char *argv[])
|
||||
{
|
||||
char hostname[32];
|
||||
|
||||
int myid, numprocs, i, j;
|
||||
double startwtime = 0.0, endwtime;
|
||||
int namelen;
|
||||
int size;
|
||||
int loop;
|
||||
MPI_Status stat;
|
||||
int sleep = 1;
|
||||
|
||||
struct timeval t_start, t_end;
|
||||
|
||||
loop = 2;
|
||||
|
||||
if (argc < 2) {
|
||||
fprintf (stderr, "Usage: %s msg_size\n", argv[0]);
|
||||
return 0;
|
||||
} else {
|
||||
size = atoi (argv[1]);
|
||||
if (argc > 2)
|
||||
loop = atoi (argv[2]);
|
||||
}
|
||||
|
||||
setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1);
|
||||
|
||||
MPI_Init (&argc, &argv);
|
||||
MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
|
||||
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
|
||||
|
||||
/* touch the data */
|
||||
for (i = 0; i < size; i++) {
|
||||
s_buf[i] = 'A' + i%26;
|
||||
}
|
||||
|
||||
gethostname(hostname, 32);
|
||||
|
||||
fprintf(stdout, "[%s:%s:%d] done with init and barrier\n",
|
||||
hostname, __FUNCTION__, __LINE__);
|
||||
fflush(stdout);
|
||||
|
||||
for (i = 0; i < loop + skip; i++) {
|
||||
if (i == skip)
|
||||
gettimeofday (&t_start, 0);
|
||||
if (myid == 0) {
|
||||
MPI_Send (s_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD);
|
||||
/*if (PONG)*/
|
||||
MPI_Recv (r_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD, &stat);
|
||||
} else {
|
||||
MPI_Recv (r_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD, &stat);
|
||||
/*if (PONG)*/
|
||||
MPI_Send (s_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD);
|
||||
}
|
||||
|
||||
if (CHECK && myid != 0) {
|
||||
for (j=0; j < size; j ++) {
|
||||
if (r_buf[j] != 'A' + j%26) {
|
||||
fprintf(stderr, "[%s:%s] error from byte %d \n",
|
||||
hostname, __FUNCTION__, j);
|
||||
break;
|
||||
} else {
|
||||
r_buf[j] = '0';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
gettimeofday (&t_end, 0);
|
||||
|
||||
fprintf(stdout, "[%s:%s:%d] done with pingpong\n",
|
||||
hostname, __FUNCTION__, __LINE__);
|
||||
fflush(stdout);
|
||||
|
||||
if (myid == 0) {
|
||||
double latency;
|
||||
latency = ((1.0e6 * t_end.tv_sec + t_end.tv_usec)
|
||||
- (1.0e6 * t_start.tv_sec + t_start.tv_usec)) / (2.0 * loop);
|
||||
fprintf(stdout, "length %d latency %8f\n",
|
||||
size, latency);
|
||||
fflush(stdout);
|
||||
}
|
||||
MPI_Finalize ();
|
||||
return 0;
|
||||
}
|
22
src/mca/ptl/gm/tests/lat.gm
Обычный файл
22
src/mca/ptl/gm/tests/lat.gm
Обычный файл
@ -0,0 +1,22 @@
|
||||
length 0 latency 20.963000
|
||||
length 1 latency 27.082500
|
||||
length 2 latency 27.089000
|
||||
length 4 latency 27.046000
|
||||
length 8 latency 27.111500
|
||||
length 16 latency 27.161500
|
||||
length 32 latency 27.232500
|
||||
length 64 latency 27.889500
|
||||
length 128 latency 29.147000
|
||||
length 256 latency 31.662000
|
||||
length 512 latency 34.797000
|
||||
length 1024 latency 42.762000
|
||||
length 2048 latency 57.954500
|
||||
length 4096 latency 77.715000
|
||||
length 8192 latency 128.157500
|
||||
length 16384 latency 306.884000
|
||||
length 32768 latency 429.529000
|
||||
length 65536 latency 666.954000
|
||||
length 131072 latency 1126.890500
|
||||
length 262144 latency 2084.019500
|
||||
length 524288 latency 4060.802500
|
||||
length 1048576 latency 8011.251500
|
22
src/mca/ptl/gm/tests/lat.gm.16k
Обычный файл
22
src/mca/ptl/gm/tests/lat.gm.16k
Обычный файл
@ -0,0 +1,22 @@
|
||||
length 0 latency 6.293000
|
||||
length 1 latency 6.914000
|
||||
length 2 latency 6.961000
|
||||
length 4 latency 6.967000
|
||||
length 8 latency 6.921000
|
||||
length 16 latency 7.002000
|
||||
length 32 latency 7.144000
|
||||
length 64 latency 7.231000
|
||||
length 128 latency 7.750000
|
||||
length 256 latency 9.867000
|
||||
length 512 latency 11.760000
|
||||
length 1024 latency 14.796000
|
||||
length 2048 latency 21.720000
|
||||
length 4096 latency 31.352000
|
||||
length 8192 latency 52.200000
|
||||
length 16384 latency 213.106000
|
||||
length 32768 latency 310.099000
|
||||
length 65536 latency 480.808000
|
||||
length 131072 latency 816.464000
|
||||
length 262144 latency 1508.072000
|
||||
length 524288 latency 2887.136000
|
||||
length 1048576 latency 5680.983000
|
48
src/mca/ptl/gm/tests/lat.gm.64k
Обычный файл
48
src/mca/ptl/gm/tests/lat.gm.64k
Обычный файл
@ -0,0 +1,48 @@
|
||||
|
||||
--------------------------------------------------
|
||||
length 0 latency 6.276000
|
||||
length 1 latency 6.863000
|
||||
length 2 latency 6.876000
|
||||
length 4 latency 6.870000
|
||||
length 8 latency 6.880000
|
||||
length 16 latency 6.994000
|
||||
length 32 latency 7.096000
|
||||
length 64 latency 7.202000
|
||||
length 128 latency 7.731000
|
||||
length 256 latency 9.877000
|
||||
length 512 latency 11.703000
|
||||
length 1024 latency 14.837000
|
||||
length 2048 latency 21.597000
|
||||
length 4096 latency 31.614000
|
||||
length 8192 latency 52.728000
|
||||
length 16384 latency 91.849000
|
||||
length 32768 latency 169.010000
|
||||
length 65536 latency 425.583000
|
||||
length 131072 latency 778.310000
|
||||
length 262144 latency 1466.517000
|
||||
length 524288 latency 2840.390000
|
||||
length 1048576 latency 5657.727000
|
||||
|
||||
---------------------------------------------------
|
||||
length 0 latency 6.249000
|
||||
length 1 latency 6.865000
|
||||
length 2 latency 6.876000
|
||||
length 4 latency 6.909000
|
||||
length 8 latency 6.903000
|
||||
length 16 latency 6.937000
|
||||
length 32 latency 7.084000
|
||||
length 64 latency 7.205000
|
||||
length 128 latency 7.700000
|
||||
length 256 latency 9.832000
|
||||
length 512 latency 11.689000
|
||||
length 1024 latency 14.779000
|
||||
length 2048 latency 21.528000
|
||||
length 4096 latency 31.625000
|
||||
length 8192 latency 52.796000
|
||||
length 16384 latency 91.786000
|
||||
length 32768 latency 168.839000
|
||||
length 65536 latency 428.174000
|
||||
length 131072 latency 783.687000
|
||||
length 262144 latency 1462.125000
|
||||
length 524288 latency 2851.882000
|
||||
length 1048576 latency 5655.458000
|
149
src/mca/ptl/gm/tests/lat.gm.dmachines
Обычный файл
149
src/mca/ptl/gm/tests/lat.gm.dmachines
Обычный файл
@ -0,0 +1,149 @@
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 0 latency 6.276500
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 1 latency 6.908500
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 2 latency 7.014000
|
||||
WARNING: session directory not removed - function not implemented
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 4 latency 6.948500
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 8 latency 6.935500
|
||||
WARNING: session directory not removed - function not implemented
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 16 latency 7.053500
|
||||
WARNING: session directory not removed - function not implemented
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 32 latency 7.104000
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 64 latency 7.323000
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 128 latency 7.733500
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 256 latency 9.846000
|
||||
WARNING: session directory not removed - function not implemented
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 512 latency 11.741500
|
||||
WARNING: session directory not removed - function not implemented
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 1024 latency 14.823500
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 2048 latency 21.604000
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 4096 latency 31.398000
|
||||
WARNING: session directory not removed - function not implemented
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 8192 latency 52.105500
|
||||
WARNING: session directory not removed - function not implemented
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
[d0-mellanox:main:86] done with pingpong
|
||||
length 16384 latency 212.413000
|
||||
[d1-mellanox:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
WARNING: session directory not removed - function not implemented
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
[d0-mellanox:main:55] done with init and barrier
|
||||
[d1-mellanox:main:55] done with init and barrier
|
||||
Terminated
|
28
src/mca/ptl/gm/tests/lat.gm.dmachines2
Обычный файл
28
src/mca/ptl/gm/tests/lat.gm.dmachines2
Обычный файл
@ -0,0 +1,28 @@
|
||||
|
||||
16k Threshold 64k Threshold
|
||||
(latency usecs) (latency usecs)
|
||||
|
||||
length 0 6.347000 6.27
|
||||
length 1 6.916000 6.86
|
||||
length 2 6.937000 6.87
|
||||
length 4 6.948000 6.87
|
||||
length 8 6.919000 6.88
|
||||
length 16 6.971000 6.99
|
||||
length 32 7.090000 7.09
|
||||
length 64 7.230000 7.20
|
||||
length 128 7.723000 7.73
|
||||
length 256 9.839000 9.87
|
||||
length 512 11.727000 11.70
|
||||
length 1024 14.816000 14.83
|
||||
length 2048 21.509000 21.59
|
||||
length 4096 31.337000 31.61
|
||||
length 8192 51.663000 52.72
|
||||
length 16384 213.005000 91.84
|
||||
length 32768 314.262000 169.01
|
||||
length 65536 485.868000 425.58
|
||||
length 131072 787.057000 783.62
|
||||
length 262144 1464.489000 1462.16
|
||||
length 524288 2871.909000 2851.17
|
||||
length 1048576 5643.708000 5655.12
|
||||
|
||||
|
198
src/mca/ptl/gm/tests/lat.gm2
Обычный файл
198
src/mca/ptl/gm/tests/lat.gm2
Обычный файл
@ -0,0 +1,198 @@
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 0 latency 16.824000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 1 latency 20.012500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 2 latency 20.039000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a1:main:86] done with pingpong
|
||||
[a0:main:86] done with pingpong
|
||||
length 4 latency 20.038000
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 8 latency 20.031500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 16 latency 20.068000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 32 latency 19.955500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 64 latency 21.302000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 128 latency 22.708000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 256 latency 24.971000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 512 latency 28.205000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 1024 latency 35.137500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 2048 latency 49.086500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 4096 latency 70.039500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 8192 latency 117.148000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 16384 latency 292.963500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a1:main:86] done with pingpong
|
||||
[a0:main:86] done with pingpong
|
||||
length 32768 latency 419.357500
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 65536 latency 654.070000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 131072 latency 1130.282500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 262144 latency 2129.726500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 524288 latency 4156.120000
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
/home/1/santhana/ompi_gm_install/lib/
|
||||
[a0:main:55] done with init and barrier
|
||||
[a1:main:55] done with init and barrier
|
||||
[a0:main:86] done with pingpong
|
||||
length 1048576 latency 8047.606500
|
||||
[a1:main:86] done with pingpong
|
||||
WARNING: session directory not removed - function not implemented
|
||||
Segmentation fault
|
16
src/mca/ptl/gm/tests/lat.sh
Исполняемый файл
16
src/mca/ptl/gm/tests/lat.sh
Исполняемый файл
@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
prog=./lat
|
||||
i=$((0))
|
||||
while [ $i -le $((1<<20)) ] ;
|
||||
do
|
||||
echo $i >&2
|
||||
#prun -N 2 -B0 $prog 1000 $i
|
||||
./run.sh $prog $i 500
|
||||
#Allow the prun to clean up
|
||||
sleep 4
|
||||
if [ $i -eq 0 ] ; then
|
||||
i=$((1))
|
||||
else
|
||||
i=$(($i*2))
|
||||
fi
|
||||
done
|
2
src/mca/ptl/gm/tests/mac
Обычный файл
2
src/mca/ptl/gm/tests/mac
Обычный файл
@ -0,0 +1,2 @@
|
||||
d0
|
||||
d1
|
30
src/mca/ptl/gm/tests/mp-ping-results
Обычный файл
30
src/mca/ptl/gm/tests/mp-ping-results
Обычный файл
@ -0,0 +1,30 @@
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
FINISHED MPI_Comm_SIZE: size is 2
|
||||
FINISHED MPI_Comm_SIZE: size is 2
|
||||
mpi-ping: ping-pong (using blocking send/recv)
|
||||
nprocs=2, reps=1000, min bytes=0, max bytes=1048576 inc bytes=0
|
||||
0 pings 1
|
||||
0 pinged 1: 0 bytes 6.28 uSec 0.00 MB/s
|
||||
0 pinged 1: 1 bytes 6.93 uSec 0.14 MB/s
|
||||
0 pinged 1: 2 bytes 6.93 uSec 0.29 MB/s
|
||||
0 pinged 1: 4 bytes 6.96 uSec 0.57 MB/s
|
||||
0 pinged 1: 8 bytes 6.94 uSec 1.15 MB/s
|
||||
0 pinged 1: 16 bytes 7.01 uSec 2.28 MB/s
|
||||
0 pinged 1: 32 bytes 7.12 uSec 4.49 MB/s
|
||||
0 pinged 1: 64 bytes 7.26 uSec 8.82 MB/s
|
||||
0 pinged 1: 128 bytes 7.74 uSec 16.54 MB/s
|
||||
0 pinged 1: 256 bytes 10.01 uSec 25.57 MB/s
|
||||
0 pinged 1: 512 bytes 11.82 uSec 43.32 MB/s
|
||||
0 pinged 1: 1024 bytes 14.98 uSec 68.34 MB/s
|
||||
0 pinged 1: 2048 bytes 21.79 uSec 93.98 MB/s
|
||||
0 pinged 1: 4096 bytes 32.14 uSec 127.44 MB/s
|
||||
0 pinged 1: 8192 bytes 53.68 uSec 152.60 MB/s
|
||||
0 pinged 1: 16384 bytes 217.31 uSec 75.40 MB/s
|
||||
0 pinged 1: 32768 bytes 319.34 uSec 102.61 MB/s
|
||||
0 pinged 1: 65536 bytes 481.60 uSec 136.08 MB/s
|
||||
0 pinged 1: 131072 bytes 824.24 uSec 159.02 MB/s
|
||||
0 pinged 1: 262144 bytes 1506.19 uSec 174.04 MB/s
|
||||
0 pinged 1: 524288 bytes 2870.90 uSec 182.62 MB/s
|
||||
0 pinged 1: 1048576 bytes 5659.85 uSec 185.27 MB/s
|
||||
WARNING: session directory not removed - function not implemented
|
Двоичные данные
src/mca/ptl/gm/tests/mpi-ping
Исполняемый файл
Двоичные данные
src/mca/ptl/gm/tests/mpi-ping
Исполняемый файл
Двоичный файл не отображается.
29
src/mca/ptl/gm/tests/mpi-ping-results
Обычный файл
29
src/mca/ptl/gm/tests/mpi-ping-results
Обычный файл
@ -0,0 +1,29 @@
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
/home/1/santhana/openmpi_gm_install_d/lib/
|
||||
FINISHED MPI_Comm_SIZE: size is 2
|
||||
mpi-ping: ping-pong (using blocking send/recv)
|
||||
nprocs=2, reps=1000, min bytes=0, max bytes=1048576 inc bytes=0
|
||||
0 pings 1
|
||||
FINISHED MPI_Comm_SIZE: size is 2
|
||||
0 pinged 1: 0 bytes 6.27 uSec 0.00 MB/s
|
||||
0 pinged 1: 1 bytes 6.93 uSec 0.14 MB/s
|
||||
0 pinged 1: 2 bytes 6.95 uSec 0.29 MB/s
|
||||
0 pinged 1: 4 bytes 6.95 uSec 0.58 MB/s
|
||||
0 pinged 1: 8 bytes 6.93 uSec 1.15 MB/s
|
||||
0 pinged 1: 16 bytes 7.04 uSec 2.27 MB/s
|
||||
0 pinged 1: 32 bytes 7.12 uSec 4.49 MB/s
|
||||
0 pinged 1: 64 bytes 7.27 uSec 8.80 MB/s
|
||||
0 pinged 1: 128 bytes 7.74 uSec 16.53 MB/s
|
||||
0 pinged 1: 256 bytes 10.01 uSec 25.56 MB/s
|
||||
0 pinged 1: 512 bytes 11.84 uSec 43.25 MB/s
|
||||
0 pinged 1: 1024 bytes 15.02 uSec 68.19 MB/s
|
||||
0 pinged 1: 2048 bytes 21.78 uSec 94.01 MB/s
|
||||
0 pinged 1: 4096 bytes 32.18 uSec 127.30 MB/s
|
||||
0 pinged 1: 8192 bytes 53.69 uSec 152.58 MB/s
|
||||
0 pinged 1: 16384 bytes 219.26 uSec 74.72 MB/s
|
||||
0 pinged 1: 32768 bytes 318.48 uSec 102.89 MB/s
|
||||
0 pinged 1: 65536 bytes 481.79 uSec 136.02 MB/s
|
||||
0 pinged 1: 131072 bytes 814.28 uSec 160.97 MB/s
|
||||
0 pinged 1: 262144 bytes 1506.43 uSec 174.02 MB/s
|
||||
0 pinged 1: 524288 bytes 2873.05 uSec 182.48 MB/s
|
||||
0 pinged 1: 1048576 bytes 5651.81 uSec 185.53 MB/s
|
404
src/mca/ptl/gm/tests/mpi-ping.c
Обычный файл
404
src/mca/ptl/gm/tests/mpi-ping.c
Обычный файл
@ -0,0 +1,404 @@
|
||||
/*
|
||||
* MPI ping program
|
||||
*
|
||||
* Patterned after the example in the Quadrics documentation
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/select.h>
|
||||
|
||||
#include <getopt.h>
|
||||
|
||||
#include "mpi.h"
|
||||
|
||||
static int str2size(char *str)
|
||||
{
|
||||
int size;
|
||||
char mod[32];
|
||||
|
||||
switch (sscanf(str, "%d%1[mMkK]", &size, mod)) {
|
||||
case 1:
|
||||
return (size);
|
||||
|
||||
case 2:
|
||||
switch (*mod) {
|
||||
case 'm':
|
||||
case 'M':
|
||||
return (size << 20);
|
||||
|
||||
case 'k':
|
||||
case 'K':
|
||||
return (size << 10);
|
||||
|
||||
default:
|
||||
return (size);
|
||||
}
|
||||
|
||||
default:
|
||||
return (-1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void usage(void)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Usage: mpi-ping [flags] <min bytes> [<max bytes>] [<inc bytes>]\n"
|
||||
" mpi-ping -h\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
|
||||
static void help(void)
|
||||
{
|
||||
printf
|
||||
("Usage: mpi-ping [flags] <min bytes> [<max bytes>] [<inc bytes>]\n"
|
||||
"\n" " Flags may be any of\n"
|
||||
" -B use blocking send/recv\n"
|
||||
" -C check data\n"
|
||||
" -O overlapping pings\n"
|
||||
" -W perform warm-up phase\n"
|
||||
" -r number repetitions to time\n"
|
||||
" -h print this info\n" "\n"
|
||||
" Numbers may be postfixed with 'k' or 'm'\n\n");
|
||||
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
MPI_Status status;
|
||||
MPI_Request recv_request;
|
||||
MPI_Request send_request;
|
||||
char *rbuf;
|
||||
char *tbuf;
|
||||
int c;
|
||||
int i;
|
||||
int bytes;
|
||||
int nproc;
|
||||
int peer;
|
||||
int proc;
|
||||
int r;
|
||||
int tag = 0x666;
|
||||
|
||||
/*
|
||||
* default options / arguments
|
||||
*/
|
||||
int reps = 10000;
|
||||
int blocking = 0;
|
||||
int check = 0;
|
||||
int overlap = 0;
|
||||
int warmup = 0;
|
||||
int inc_bytes = 0;
|
||||
int max_bytes = 0;
|
||||
int min_bytes = 0;
|
||||
|
||||
setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1);
|
||||
|
||||
MPI_Init(&argc, &argv);
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
|
||||
|
||||
printf("FINISHED MPI_Comm_SIZE: size is %d\n",nproc);
|
||||
fflush(stdout);
|
||||
|
||||
while ((c = getopt(argc, argv, "BCOWr:h")) != -1) {
|
||||
switch (c) {
|
||||
|
||||
case 'B':
|
||||
blocking = 1;
|
||||
break;
|
||||
|
||||
case 'C':
|
||||
check = 1;
|
||||
break;
|
||||
|
||||
case 'O':
|
||||
overlap = 1;
|
||||
break;
|
||||
|
||||
case 'W':
|
||||
warmup = 1;
|
||||
break;
|
||||
|
||||
case 'r':
|
||||
if ((reps = str2size(optarg)) <= 0) {
|
||||
usage();
|
||||
}
|
||||
break;
|
||||
|
||||
case 'h':
|
||||
help();
|
||||
|
||||
default:
|
||||
usage();
|
||||
}
|
||||
}
|
||||
|
||||
if (optind == argc) {
|
||||
min_bytes = 0;
|
||||
} else if ((min_bytes = str2size(argv[optind++])) < 0) {
|
||||
usage();
|
||||
}
|
||||
|
||||
if (optind == argc) {
|
||||
max_bytes = min_bytes;
|
||||
} else if ((max_bytes = str2size(argv[optind++])) < min_bytes) {
|
||||
usage();
|
||||
}
|
||||
|
||||
if (optind == argc) {
|
||||
inc_bytes = 0;
|
||||
} else if ((inc_bytes = str2size(argv[optind++])) < 0) {
|
||||
usage();
|
||||
}
|
||||
|
||||
if (nproc == 1) {
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
|
||||
if ((rbuf = (char *) malloc(max_bytes ? max_bytes : 8)) == NULL) {
|
||||
perror("malloc");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if ((tbuf = (char *) malloc(max_bytes ? max_bytes : 8)) == NULL) {
|
||||
perror("malloc");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (check) {
|
||||
for (i = 0; i < max_bytes; i++) {
|
||||
tbuf[i] = i & 255;
|
||||
rbuf[i] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (proc == 0) {
|
||||
if (overlap) {
|
||||
printf("mpi-ping: overlapping ping-pong\n");
|
||||
} else if (blocking) {
|
||||
printf("mpi-ping: ping-pong (using blocking send/recv)\n");
|
||||
} else {
|
||||
printf("mpi-ping: ping-pong\n");
|
||||
}
|
||||
if (check) {
|
||||
printf("data checking enabled\n");
|
||||
}
|
||||
printf("nprocs=%d, reps=%d, min bytes=%d, max bytes=%d inc bytes=%d\n",
|
||||
nproc, reps, min_bytes, max_bytes, inc_bytes);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
peer = proc ^ 1;
|
||||
|
||||
if ((peer < nproc) && (peer & 1)) {
|
||||
printf("%d pings %d\n", proc, peer);
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
if (warmup) {
|
||||
|
||||
if (proc == 0) {
|
||||
puts("warm-up phase");
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
for (r = 0; r < reps; r++) {
|
||||
if (peer >= nproc) {
|
||||
break;
|
||||
}
|
||||
MPI_Irecv(rbuf, max_bytes, MPI_BYTE, peer, tag, MPI_COMM_WORLD,
|
||||
&recv_request);
|
||||
MPI_Isend(tbuf, max_bytes, MPI_BYTE, peer, tag, MPI_COMM_WORLD,
|
||||
&send_request);
|
||||
MPI_Wait(&send_request, &status);
|
||||
MPI_Wait(&recv_request, &status);
|
||||
}
|
||||
|
||||
if (proc == 0) {
|
||||
puts("warm-up phase done");
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
/*
|
||||
* Main loop
|
||||
*/
|
||||
|
||||
for (bytes = min_bytes; bytes <= max_bytes;
|
||||
bytes = inc_bytes ? bytes + inc_bytes : bytes ? 2 * bytes : 1) {
|
||||
|
||||
double t = 0.0;
|
||||
double tv[2];
|
||||
|
||||
r = reps;
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
if (peer < nproc) {
|
||||
|
||||
if (overlap) {
|
||||
|
||||
/*
|
||||
* MPI_Isend / MPI_Irecv overlapping ping-pong
|
||||
*/
|
||||
|
||||
tv[0] = MPI_Wtime();
|
||||
|
||||
for (r = 0; r < reps; r++) {
|
||||
|
||||
MPI_Irecv(rbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD, &recv_request);
|
||||
MPI_Isend(tbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD, &send_request);
|
||||
MPI_Wait(&send_request, &status);
|
||||
MPI_Wait(&recv_request, &status);
|
||||
|
||||
if (check) {
|
||||
for (i = 0; i < bytes; i++) {
|
||||
if (rbuf[i] != (i & 255)) {
|
||||
puts("mpi-ping: Error: Invalid data received");
|
||||
}
|
||||
rbuf[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tv[1] = MPI_Wtime();
|
||||
|
||||
} else if (blocking) {
|
||||
|
||||
/*
|
||||
* MPI_Send / MPI_Recv ping-pong
|
||||
*/
|
||||
|
||||
tv[0] = MPI_Wtime();
|
||||
|
||||
if (peer < nproc) {
|
||||
if (proc & 1) {
|
||||
r--;
|
||||
MPI_Recv(rbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD, &status);
|
||||
|
||||
if (check) {
|
||||
for (i = 0; i < bytes; i++) {
|
||||
if (rbuf[i] != (i & 255)) {
|
||||
puts("mpi-ping: Error: Invalid data received");
|
||||
}
|
||||
rbuf[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (r-- > 0) {
|
||||
|
||||
MPI_Send(tbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD);
|
||||
MPI_Recv(rbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD, &status);
|
||||
|
||||
if (check) {
|
||||
for (i = 0; i < bytes; i++) {
|
||||
if (rbuf[i] != (i & 255)) {
|
||||
puts("mpi-ping: Error: Invalid data received");
|
||||
}
|
||||
rbuf[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (proc & 1) {
|
||||
MPI_Send(tbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD);
|
||||
}
|
||||
}
|
||||
|
||||
tv[1] = MPI_Wtime();
|
||||
|
||||
} else {
|
||||
|
||||
/*
|
||||
* MPI_Isend / MPI_Irecv ping-pong
|
||||
*/
|
||||
|
||||
tv[0] = MPI_Wtime();
|
||||
|
||||
if (peer < nproc) {
|
||||
if (proc & 1) {
|
||||
r--;
|
||||
MPI_Irecv(rbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD, &recv_request);
|
||||
MPI_Wait(&recv_request, &status);
|
||||
|
||||
if (check) {
|
||||
for (i = 0; i < bytes; i++) {
|
||||
if (rbuf[i] != (i & 255)) {
|
||||
puts("mpi-ping: Error: Invalid data received");
|
||||
}
|
||||
rbuf[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (r-- > 0) {
|
||||
|
||||
MPI_Isend(tbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD, &send_request);
|
||||
MPI_Wait(&send_request, &status);
|
||||
MPI_Irecv(rbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD, &recv_request);
|
||||
MPI_Wait(&recv_request, &status);
|
||||
|
||||
if (check) {
|
||||
for (i = 0; i < bytes; i++) {
|
||||
if (rbuf[i] != (i & 255)) {
|
||||
puts("mpi-ping: Error: Invalid data received");
|
||||
}
|
||||
rbuf[i] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (proc & 1) {
|
||||
MPI_Isend(tbuf, bytes, MPI_BYTE, peer, tag,
|
||||
MPI_COMM_WORLD, &send_request);
|
||||
MPI_Wait(&send_request, &status);
|
||||
}
|
||||
}
|
||||
|
||||
tv[1] = MPI_Wtime();
|
||||
}
|
||||
|
||||
/*
|
||||
* Calculate time interval in useconds (half round trip)
|
||||
*/
|
||||
|
||||
t = (tv[1] - tv[0]) * 1000000.0 / (2 * reps);
|
||||
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
|
||||
if ((peer < nproc) && (peer & 1)) {
|
||||
printf("%3d pinged %3d: %8d bytes %9.2f uSec %8.2f MB/s\n",
|
||||
proc, peer, bytes, t, bytes / (t));
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
|
||||
MPI_Barrier(MPI_COMM_WORLD);
|
||||
MPI_Finalize();
|
||||
|
||||
return EXIT_SUCCESS;
|
||||
}
|
1
src/mca/ptl/gm/tests/run.sh
Исполняемый файл
1
src/mca/ptl/gm/tests/run.sh
Исполняемый файл
@ -0,0 +1 @@
|
||||
mpirun -np 2 -hostfile mac -- ${PWD}/$1 $2 $3
|
Загрузка…
Ссылка в новой задаче
Block a user