ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Several bugs have been fixed and the functionality of the GM PTL has been improved:

Просмотр исходного кода 
- Now we support GM 1.x
- We detect correctly the version of GM and activate some defines
- memory leaks fixes (dont release a fragment several times and release all fragments)
- protect the free_list while debug is activated
- change the place where we keep the informations about the local node as well as for the peers.
  It should be a little bit cleaner as we have less copies.
- first step toward supporting OSes without memory registration.
- correctly select the communication protocol depending on the capabilities of the GM library.
  There is still some work to be done as the detection is ... let's say user based at this moment (using
  MCA parameters).

This commit was SVN r5248.
Этот коммит содержится в:
George Bosilca 2005-04-10 16:20:04 +00:00
родитель 2bc6663672
Коммит 247e271b14
6 изменённых файлов: 131 добавлений и 98 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

34
src/mca/ptl/gm/src/ptl_gm.c
Просмотреть файл

@ -19,6 +19,7 @@
*/
#include "ompi_config.h"
#include "gm_config.h"
#include <string.h>
#include "class/ompi_bitmap.h"
#include "util/output.h"
@ -28,8 +29,8 @@
#include "mca/ptl/base/ptl_base_header.h"
#include "ptl_gm.h"
#include "ptl_gm_proc.h"
#include "ptl_gm_peer.h"
#include "ptl_gm_priv.h"
#include "ptl_gm_peer.h"
#include "ptl_gm_sendfrag.h"
mca_ptl_gm_module_t mca_ptl_gm_module = {
@ -75,7 +76,6 @@ mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
struct ompi_proc_t *orte_proc;
mca_ptl_gm_proc_t *ptl_proc;
mca_ptl_gm_peer_t *ptl_peer;
unsigned int lid;
ompi_proc_t* local_proc = ompi_proc_local();
for (i = 0; i < nprocs; i++) {
@ -102,20 +102,28 @@ mca_ptl_gm_add_procs (struct mca_ptl_base_module_t *ptl,
ptl_peer->peer_ptl = (mca_ptl_gm_module_t *) ptl;
ptl_peer->peer_proc = ptl_proc;
ptl_peer->global_id = ptl_proc->proc_addrs->global_id;
ptl_peer->port_number = ptl_proc->proc_addrs->port_id;
if (GM_SUCCESS !=
gm_global_id_to_node_id (((mca_ptl_gm_module_t *) ptl)->gm_port,
ptl_proc->proc_addrs[j].global_id,
&lid)) {
ptl_peer->peer_addr.port_id = ptl_proc->proc_addrs->port_id;
#if GM_API_VERSION > 0x200
ptl_peer->peer_addr.global_id = ptl_proc->proc_addrs->global_id;
if (GM_SUCCESS != gm_global_id_to_node_id(((mca_ptl_gm_module_t *) ptl)->gm_port,
ptl_proc->proc_addrs[j].global_id,
&(ptl_peer->peer_addr.local_id))) {
ompi_output( 0, "[%s:%d] error in converting global to local id \n",
__FILE__, __LINE__ );
return OMPI_ERR_BAD_PARAM;
}
ptl_peer->local_id = lid;
#else
strncpy( ptl_peer->peer_addr.global_id, ptl_proc->proc_addrs->global_id, GM_MAX_HOST_NAME_LEN );
ptl_peer->peer_addr.local_id = gm_host_name_to_node_id( ((mca_ptl_gm_module_t *) ptl)->gm_port,
ptl_proc->proc_addrs[j].global_id );
if( GM_NO_SUCH_NODE_ID == ptl_peer->peer_addr.local_id ) {
ompi_output( 0, "Unable to convert the remote host name (%s) to a host id",
ptl_proc->proc_addrs[j].global_id );
return OMPI_ERR_BAD_PARAM;
}
#endif /* GM_API_VERSION > 0x200 */
ptl_proc->peer_arr[ptl_proc->proc_peer_count] = ptl_peer;
ptl_proc->proc_peer_count++;
ptl_peer->peer_addr = ptl_proc->proc_addrs + i;
}
ompi_bitmap_set_bit (reachable, i);
OMPI_THREAD_UNLOCK (&ptl_proc->proc_lock);
@ -307,7 +315,7 @@ static void mca_ptl_gm_basic_ack_callback( struct gm_port* port, void* context,
frag_base = (mca_ptl_base_frag_t*)header->hdr_ack.hdr_dst_addr.pval;
gm_ptl = (mca_ptl_gm_module_t *)frag_base->frag_owner;
OMPI_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
OMPI_GM_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
/* release the send token */
ompi_atomic_add( &(gm_ptl->num_send_tokens), 1 );
}
@ -358,8 +366,8 @@ mca_ptl_gm_matched( mca_ptl_base_module_t * ptl,
gm_send_with_callback( ((mca_ptl_gm_module_t*)ptl)->gm_port, hdr,
GM_SIZE, sizeof(mca_ptl_base_ack_header_t),
GM_LOW_PRIORITY,
peer->local_id,
peer->port_number,
peer->peer_addr.local_id,
peer->peer_addr.port_id,
mca_ptl_gm_basic_ack_callback,
(void *)hdr );
}

24
src/mca/ptl/gm/src/ptl_gm.h
Просмотреть файл

@ -23,11 +23,14 @@
#ifndef MCA_PTL_GM_H
#define MCA_PTL_GM_H
#include "ompi_config.h"
#include "gm_config.h"
#include "class/ompi_free_list.h"
#include "mca/ptl/ptl.h"
#include "ptl_gm_priv.h"
#include "ptl_gm_peer.h"
#define MCA_PTL_GM_STATISTICS 0
#define THRESHOLD 16384
#define MAX_RECV_TOKENS 256
#define PTL_GM_ADMIN_SEND_TOKENS 0
#define PTL_GM_ADMIN_RECV_TOKENS 0
@ -76,9 +79,7 @@ extern "C" {
struct mca_ptl_gm_module_t {
mca_ptl_base_module_t super; /**< base PTL module interface */
struct gm_port *gm_port;
unsigned int local_id;
unsigned int global_id;
unsigned int port_id;
mca_ptl_gm_addr_t local_addr;
unsigned int num_send_tokens;
unsigned int num_recv_tokens;
unsigned int max_send_tokens;
@ -99,7 +100,7 @@ extern "C" {
#if MCA_PTL_GM_STATISTICS
size_t ptl_bytes_sent;
size_t ptl_bytes_recv;
#endif
#endif /* MCA_PTL_GM_STATISTICS */
};
typedef struct mca_ptl_gm_module_t mca_ptl_gm_module_t;
@ -226,6 +227,19 @@ extern "C" {
union mca_ptl_base_header_t;
void mca_ptl_gm_dump_header( char* str, union mca_ptl_base_header_t* hdr );
#if OMPI_ENABLE_DEBUG
#include "class/ompi_list.h"
/* If debug is enabled we have to work around the item validity checks. */
#define OMPI_GM_FREE_LIST_RETURN( LIST, ITEM ) \
do { \
(ITEM)->ompi_list_item_refcount = 0; \
(ITEM)->ompi_list_item_lock.u.lock = 0; \
OMPI_FREE_LIST_RETURN( (LIST), (ITEM) ); \
} while(0)
#else
#define OMPI_GM_FREE_LIST_RETURN OMPI_FREE_LIST_RETURN
#endif /* OMPI_ENABLE_DEBUG */
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif

63
src/mca/ptl/gm/src/ptl_gm_component.c
Просмотреть файл

@ -213,9 +213,13 @@ mca_ptl_gm_module_store_data_toexchange (void)
for (i = 0; i < mca_ptl_gm_component.gm_num_ptl_modules; i++) {
mca_ptl_gm_module_t *ptl = mca_ptl_gm_component.gm_ptl_modules[i];
addrs[i].local_id = ptl->local_id;
addrs[i].global_id = ptl->global_id;
addrs[i].port_id = ptl->port_id;
addrs[i].local_id = ptl->local_addr.local_id;
#if GM_API_VERSION > 0x200
addrs[i].global_id = ptl->local_addr.global_id;
#else
strncpy( addrs[i].global_id, ptl->local_addr.global_id, GM_MAX_HOST_NAME_LEN );
#endif /* GM_API_VERSION > 0x200 */
addrs[i].port_id = ptl->local_addr.port_id;
}
rc = mca_base_modex_send (&mca_ptl_gm_component.super.ptlm_version, addrs,
size);
@ -251,10 +255,13 @@ static int32_t
mca_ptl_gm_discover_boards( mca_ptl_gm_module_t** pptl,
uint32_t max_ptls, uint32_t max_boards, uint32_t max_port )
{
uint32_t board_no, port_no;
uint32_t board_no, port_no, index = 0, local_id;
struct gm_port* gm_port;
uint32_t index = 0;
uint32_t local_id, global_id;
#if GM_API_VERSION > 0x200
int32_t global_id;
#else
char global_id[GM_MAX_HOST_NAME_LEN];
#endif /* GM_API_VERSION > 0x200 */
for( board_no = 0; board_no < max_boards; board_no++ ) {
@ -278,22 +285,34 @@ mca_ptl_gm_discover_boards( mca_ptl_gm_module_t** pptl,
ompi_output (0, " failure to get local_id \n");
continue;
}
/* Convert local id to global id */
/* Gather an unique id for the node */
#if GM_API_VERSION > 0x200
if (GM_SUCCESS != gm_node_id_to_global_id( gm_port, local_id, &global_id) ) {
ompi_output (0, " Error: Unable to get my GM global id \n");
continue;
}
ompi_output (0, " Error: Unable to get my GM global unique id \n");
continue;
}
#else
{
if( GM_SUCCESS != gm_get_host_name( gm_port, global_id ) ) {
ompi_output( 0, "Error: Unable to get the GM host name\n" );
continue;
}
}
#endif /* GM_API_VERSION > 0x200 */
/* Create the ptl. If fail return the number of already created */
if( OMPI_SUCCESS != mca_ptl_gm_create( &(pptl[index]) ) ) {
return index;
}
pptl[index]->port_id = port_no;
pptl[index]->gm_port = gm_port;
pptl[index]->local_id = local_id;
pptl[index]->global_id = global_id;
pptl[index]->gm_port = gm_port;
pptl[index]->local_addr.port_id = port_no;
pptl[index]->local_addr.local_id = local_id;
#if GM_API_VERSION > 0x200
pptl[index]->local_addr.global_id = global_id;
#else
strncpy( pptl[index]->local_addr.global_id, global_id, GM_MAX_HOST_NAME_LEN );
#endif /* GM_API_VERSION > 0x200 */
/* everything is OK let's mark it as usable and go to the next one */
if( (++index) >= max_ptls ) {
@ -359,10 +378,10 @@ mca_ptl_gm_init_sendrecv (mca_ptl_gm_module_t * ptl)
}
for (i = 0; i < ptl->num_send_tokens; i++) {
sfragment->send_buf = NULL;
OMPI_FREE_LIST_RETURN( &(ptl->gm_send_frags), (ompi_list_item_t *)sfragment );
OMPI_FREE_LIST_RETURN( &(ptl->gm_send_dma_frags),
(ompi_list_item_t *)((char*)ptl->gm_send_dma_memory +
i * mca_ptl_gm_component.gm_segment_size) );
OMPI_GM_FREE_LIST_RETURN( &(ptl->gm_send_frags), (ompi_list_item_t*)sfragment );
OMPI_GM_FREE_LIST_RETURN( &(ptl->gm_send_dma_frags),
(ompi_list_item_t*)((char*)ptl->gm_send_dma_memory +
i * mca_ptl_gm_component.gm_segment_size) );
sfragment++;
}
@ -399,14 +418,14 @@ mca_ptl_gm_init_sendrecv (mca_ptl_gm_module_t * ptl)
}
for( i = 0; i < 2; i++ ) {
OMPI_FREE_LIST_RETURN( &(ptl->gm_recv_frags_free), (ompi_list_item_t *)free_rfragment );
OMPI_GM_FREE_LIST_RETURN( &(ptl->gm_recv_frags_free), (ompi_list_item_t *)free_rfragment );
free_rfragment++;
gm_provide_receive_buffer( ptl->gm_port, (char*)ptl->gm_recv_dma_memory + i * mca_ptl_gm_component.gm_segment_size,
GM_SIZE, GM_HIGH_PRIORITY );
}
for( i = 2; i < ptl->num_recv_tokens; i++ ) {
OMPI_FREE_LIST_RETURN( &(ptl->gm_recv_frags_free), (ompi_list_item_t *)free_rfragment );
OMPI_GM_FREE_LIST_RETURN( &(ptl->gm_recv_frags_free), (ompi_list_item_t *)free_rfragment );
free_rfragment++;
gm_provide_receive_buffer( ptl->gm_port, (char*)ptl->gm_recv_dma_memory + i * mca_ptl_gm_component.gm_segment_size,
@ -547,7 +566,7 @@ char* gm_get_local_buffer( void )
void gm_release_local_buffer( char* ptr )
{
OMPI_FREE_LIST_RETURN( &(mca_ptl_gm_component.gm_unexpected_frags_data), (ompi_list_item_t*)ptr );
OMPI_GM_FREE_LIST_RETURN( &(mca_ptl_gm_component.gm_unexpected_frags_data), (ompi_list_item_t*)ptr );
}
/*

9
src/mca/ptl/gm/src/ptl_gm_peer.h
Просмотреть файл

@ -34,7 +34,11 @@ extern "C" {
* Structure used to publish GM id information to peers.
*/
struct mca_ptl_gm_addr_t {
#if GM_API_VERSION > 0x200
unsigned int global_id;
#else
char global_id[GM_MAX_HOST_NAME_LEN];
#endif /* GM_API_VERSION > 0x200 */
unsigned int local_id;
unsigned int port_id;
};
@ -48,10 +52,7 @@ struct mca_ptl_gm_peer_t {
ompi_list_item_t super;
struct mca_ptl_gm_module_t* peer_ptl;
struct mca_ptl_gm_proc_t* peer_proc;
struct mca_ptl_gm_addr_t* peer_addr; /**< address of peer */
unsigned int global_id;
unsigned int port_number;
unsigned int local_id;
struct mca_ptl_gm_addr_t peer_addr; /**< address of peer */
int num_credits;
int max_credits;
int resending;

98
src/mca/ptl/gm/src/ptl_gm_priv.c
Просмотреть файл

@ -29,40 +29,6 @@
#include "ptl_gm_priv.h"
#include "mca/pml/teg/src/pml_teg_proc.h"
static void send_continue_callback( struct gm_port *port, void * context, gm_status_t status )
{
mca_ptl_gm_module_t* gm_ptl;
mca_ptl_gm_send_frag_t* frag;
mca_ptl_base_header_t* header;
header = (mca_ptl_base_header_t*)context;
frag = header->hdr_frag.hdr_src_ptr.pval;
gm_ptl = (mca_ptl_gm_module_t *)frag->frag_send.frag_base.frag_owner;
switch( status ) {
case GM_SUCCESS:
/*frag->frag_bytes_validated += header->hdr_frag.hdr_frag_length;*/
OMPI_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
/* release the send token */
ompi_atomic_add( &(gm_ptl->num_send_tokens), 1 );
if( frag->frag_bytes_validated >= frag->frag_send.frag_base.frag_size ) {
OMPI_FREE_LIST_RETURN( &(gm_ptl->gm_send_frags), ((ompi_list_item_t*)frag) );
}
break;
case GM_SEND_TIMED_OUT:
ompi_output( 0, "send_continue timed out\n" );
break;
case GM_SEND_DROPPED:
ompi_output( 0, "send_continue dropped\n" );
break;
default:
ompi_output( 0, "send_continue other error %d\n", status );
}
}
static void mca_ptl_gm_basic_frag_callback( struct gm_port* port, void* context, gm_status_t status )
{
mca_ptl_gm_module_t* gm_ptl;
@ -74,9 +40,21 @@ static void mca_ptl_gm_basic_frag_callback( struct gm_port* port, void* context,
frag_base = (mca_ptl_base_frag_t*)header->hdr_frag.hdr_src_ptr.pval;
gm_ptl = (mca_ptl_gm_module_t *)frag_base->frag_owner;
OMPI_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
/* release the send token */
ompi_atomic_add( &(gm_ptl->num_send_tokens), 1 );
switch( status ) {
case GM_SUCCESS:
OMPI_GM_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
/* release the send token */
ompi_atomic_add( &(gm_ptl->num_send_tokens), 1 );
break;
case GM_SEND_TIMED_OUT:
ompi_output( 0, "send_continue timed out\n" );
break;
case GM_SEND_DROPPED:
ompi_output( 0, "send_continue dropped\n" );
break;
default:
ompi_output( 0, "send_continue other error %d\n", status );
}
}
static void mca_ptl_gm_get_callback( struct gm_port *port, void * context, gm_status_t status );
@ -98,9 +76,12 @@ int mca_ptl_gm_receiver_advance_pipeline( mca_ptl_gm_recv_frag_t* frag, int only
get_line = &(frag->pipeline.lines[frag->pipeline.pos_transfert]);
if( (PTL_GM_PIPELINE_TRANSFERT & get_line->flags) == PTL_GM_PIPELINE_TRANSFERT ) {
peer->get_started = true;
#if OMPI_MCA_PTL_GM_HAVE_RDMA_GET
gm_get( peer->peer_ptl->gm_port, get_line->remote_memory.lval,
get_line->local_memory.pval, get_line->length,
GM_LOW_PRIORITY, peer->local_id, peer->port_number, mca_ptl_gm_get_callback, frag );
GM_LOW_PRIORITY, peer->peer_addr.local_id, peer->peer_addr.port_id,
mca_ptl_gm_get_callback, frag );
#endif /* OMPI_MCA_PTL_GM_HAVE_RDMA_GET */
get_line->flags ^= PTL_GM_PIPELINE_REMOTE;
DO_DEBUG( count += sprintf( buffer + count, " start get %lld (%d)", get_line->length, frag->pipeline.pos_transfert ); );
frag->pipeline.pos_transfert = (frag->pipeline.pos_transfert + 1) % GM_PIPELINE_DEPTH;
@ -195,8 +176,8 @@ int mca_ptl_gm_sender_advance_pipeline( mca_ptl_gm_send_frag_t* frag )
gm_send_with_callback( peer->peer_ptl->gm_port, hdr,
GM_SIZE, sizeof(mca_ptl_gm_frag_header_t),
GM_HIGH_PRIORITY, peer->local_id, peer->port_number,
send_continue_callback, (void*)hdr );
GM_HIGH_PRIORITY, peer->peer_addr.local_id, peer->peer_addr.port_id,
mca_ptl_gm_basic_frag_callback, (void*)hdr );
send_line->flags ^= PTL_GM_PIPELINE_REMOTE;
frag->pipeline.pos_transfert = (frag->pipeline.pos_transfert + 1) % GM_PIPELINE_DEPTH;
@ -290,7 +271,7 @@ int mca_ptl_gm_send_internal_rndv_header( mca_ptl_gm_peer_t *ptl_peer,
hdr->hdr_frag.hdr_frag_offset, hdr->hdr_frag.hdr_frag_length, max_data ); );
gm_send_with_callback( ptl_peer->peer_ptl->gm_port, hdr, GM_SIZE,
sizeof(mca_ptl_gm_frag_header_t) + max_data,
GM_LOW_PRIORITY, ptl_peer->local_id, ptl_peer->port_number,
GM_LOW_PRIORITY, ptl_peer->peer_addr.local_id, ptl_peer->peer_addr.port_id,
mca_ptl_gm_basic_frag_callback, (void *)hdr );
fragment->frag_bytes_processed += max_data;
fragment->frag_bytes_validated += max_data;
@ -346,8 +327,8 @@ int mca_ptl_gm_send_burst_data( mca_ptl_gm_peer_t *ptl_peer,
/* for the last piece set the header type to FIN */
gm_send_with_callback( ptl_peer->peer_ptl->gm_port, hdr, GM_SIZE,
iov.iov_len + sizeof(mca_ptl_base_frag_header_t),
GM_LOW_PRIORITY, ptl_peer->local_id, ptl_peer->port_number,
send_continue_callback, (void*)hdr );
GM_LOW_PRIORITY, ptl_peer->peer_addr.local_id, ptl_peer->peer_addr.port_id,
mca_ptl_gm_basic_frag_callback, (void*)hdr );
hdr = NULL; /* force to retrieve a new one on the next loop */
}
DO_DEBUG( ompi_output( 0, "sender after burst offset %lld, processed %lld, validated %lld\n",
@ -383,13 +364,19 @@ int mca_ptl_gm_peer_send_continue( mca_ptl_gm_peer_t *ptl_peer,
remaining_bytes = fragment->frag_send.frag_base.frag_size - fragment->frag_bytes_processed;
if( remaining_bytes < mca_ptl_gm_component.gm_eager_limit ) {
burst_length = remaining_bytes;
} else if( remaining_bytes < mca_ptl_gm_component.gm_rndv_burst_limit ) {
burst_length = remaining_bytes % (mca_ptl_gm_component.gm_segment_size - sizeof(mca_ptl_base_frag_header_t));
} else {
if( mca_ptl_gm_component.gm_rdma_frag_size == UINT_MAX )
burst_length = 0;
else
burst_length = remaining_bytes % mca_ptl_gm_component.gm_rdma_frag_size;
#if OMPI_MCA_PTL_GM_HAVE_RDMA_GET
if( remaining_bytes < mca_ptl_gm_component.gm_rndv_burst_limit ) {
burst_length = remaining_bytes % (mca_ptl_gm_component.gm_segment_size - sizeof(mca_ptl_base_frag_header_t));
} else {
if( mca_ptl_gm_component.gm_rdma_frag_size == UINT_MAX )
burst_length = 0;
else
burst_length = remaining_bytes % mca_ptl_gm_component.gm_rdma_frag_size;
}
#else
burst_length = remaining_bytes % (mca_ptl_gm_component.gm_segment_size - sizeof(mca_ptl_base_frag_header_t));
#endif /* OMPI_MCA_PTL_GM_HAVE_RDMA_GET */
}
if( burst_length > 0 ) {
@ -404,6 +391,7 @@ int mca_ptl_gm_peer_send_continue( mca_ptl_gm_peer_t *ptl_peer,
ptl_peer->peer_ptl->super.ptl_send_progress( (mca_ptl_base_module_t*)ptl_peer->peer_ptl,
fragment->frag_send.frag_request,
(*size) );
OMPI_FREE_LIST_RETURN( &(ptl_peer->peer_ptl->gm_send_frags), ((ompi_list_item_t*)fragment) );
}
return OMPI_SUCCESS;
}
@ -418,8 +406,11 @@ int mca_ptl_gm_peer_send_continue( mca_ptl_gm_peer_t *ptl_peer,
* with the others informations. When we reach this point the rendez-vous protocol
* has already been realized so we know that the receiver expect our message.
*/
#if OMPI_MCA_PTL_GM_HAVE_RDMA_GET == 0
/* Trigger the long rendez-vous protocol only if gm_get is supported */
if( remaining_bytes > mca_ptl_gm_component.gm_rndv_burst_limit )
flags |= PTL_FLAG_GM_REQUIRE_LOCK;
#endif /* OMPI_MCA_PTL_GM_HAVE_RDMA_GET */
mca_ptl_gm_send_internal_rndv_header( ptl_peer, fragment, hdr, flags );
if( !(PTL_FLAG_GM_REQUIRE_LOCK & flags) )
return OMPI_SUCCESS;
@ -457,7 +448,7 @@ static void send_match_callback( struct gm_port* port, void* context, gm_status_
gm_ptl = (mca_ptl_gm_module_t*)((long)header->hdr_rndv.hdr_frag_length);
OMPI_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
OMPI_GM_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
/* release the send token */
ompi_atomic_add( &(gm_ptl->num_send_tokens), 1 );
}
@ -533,8 +524,7 @@ int mca_ptl_gm_peer_send( struct mca_ptl_base_module_t* ptl,
/* Send the first fragment */
gm_send_with_callback(ptl_peer->peer_ptl->gm_port, hdr,
GM_SIZE, size_out, GM_LOW_PRIORITY,
ptl_peer->local_id,
ptl_peer->port_number,
ptl_peer->peer_addr.local_id, ptl_peer->peer_addr.port_id,
send_match_callback, (void *)hdr );
if( !(flags & MCA_PTL_FLAGS_ACK) ) {
@ -649,7 +639,7 @@ static void recv_short_callback( struct gm_port* port, void* context, gm_status_
frag_base = (mca_ptl_base_frag_t*)header->hdr_dst_match.pval;
gm_ptl = (mca_ptl_gm_module_t *)frag_base->frag_owner;
OMPI_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
OMPI_GM_FREE_LIST_RETURN( &(gm_ptl->gm_send_dma_frags), ((ompi_list_item_t*)header) );
/* release the send token */
ompi_atomic_add( &(gm_ptl->num_send_tokens), 1 );
}
@ -676,7 +666,7 @@ static int mca_ptl_gm_send_quick_fin_message( struct mca_ptl_gm_peer_t* ptl_peer
gm_send_with_callback(ptl_peer->peer_ptl->gm_port, hdr,
GM_SIZE, sizeof(mca_ptl_base_ack_header_t),
GM_HIGH_PRIORITY, ptl_peer->local_id, ptl_peer->port_number,
GM_HIGH_PRIORITY, ptl_peer->peer_addr.local_id, ptl_peer->peer_addr.port_id,
recv_short_callback, (void*)hdr );
DO_DEBUG( ompi_output( 0, "receiver %p send quick message for length %lld", frag, frag->frag_header.hdr_frag.hdr_frag_length ); )
return OMPI_SUCCESS;

1
src/mca/ptl/gm/src/ptl_gm_priv.h
Просмотреть файл

@ -22,6 +22,7 @@
struct mca_ptl_gm_send_frag_t;
struct mca_ptl_gm_peer_t;
struct mca_ptl_gm_module_t;
/* Pinning down memory pages is a costly operation. We can avoid it by using a LRU list
* of pinned down memory, managed inside the GM PTL.