ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

ugni btl update. changes:

Просмотр исходного кода 
- re-enable sendi
 - move smsg common code into btl_ugni_smsg.h
 - added new parameters for smsg/eager frags
 - use get for frags larger than the smsg_limit
 - bug fixes
 - code cleanup

This commit was SVN r25897.
Этот коммит содержится в:
Nathan Hjelm 2012-02-10 00:47:29 +00:00
родитель 05a9727aca
Коммит 0ccfd3e6db
15 изменённых файлов: 720 добавлений и 572 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
ompi/mca/btl/ugni/Makefile.am
Просмотреть файл

@ -35,7 +35,8 @@ ugni_SOURCES = \
btl_ugni_send.c \
btl_ugni_put.c \
btl_ugni_get.c \
btl_ugni.h
btl_ugni.h \
btl_ugni_smsg.h
mcacomponentdir = $(pkglibdir)
mcacomponent_LTLIBRARIES = $(component_install)

55
ompi/mca/btl/ugni/btl_ugni.h
Просмотреть файл

@ -47,19 +47,8 @@
#define MCA_BTL_UGNI_CONNECT_DIRECTED_ID 0x6b61686e00000000ull
#define MCA_BTL_UGNI_DATAGRAM_MASK 0xffffffff00000000ull
typedef enum {
MCA_BTL_UGNI_TAG_SEND,
MCA_BTL_UGNI_TAG_DISCONNECT,
MCA_BTL_UGNI_TAG_PUT_INIT,
MCA_BTL_UGNI_TAG_PUT_COMPLETE
} mca_btl_ugni_smsg_tag_t;
/* Maximum number of outstanding eager messages */
extern int mca_btl_ugni_smsg_max_credits;
extern int mca_btl_ugni_smsg_mbox_size;
struct mca_btl_ugni_module_t {
mca_btl_base_module_t super;
typedef struct mca_btl_ugni_module_t {
mca_btl_base_module_t super;
ompi_common_ugni_device_t *device;
@ -76,8 +65,6 @@ struct mca_btl_ugni_module_t {
gni_cq_handle_t bte_local_cq;
gni_cq_handle_t smsg_remote_cq;
};
typedef struct mca_btl_ugni_module_t mca_btl_ugni_module_t;
struct mca_btl_ugni_component_t {
/* base BTL component */
@ -93,31 +80,38 @@ struct mca_btl_ugni_component_t {
/* ugni modules */
mca_btl_ugni_module_t *modules;
/* eager send limit in bytes */
/* used as the threshold for switching from SMSG */
size_t eager_limit;
size_t smsg_max_data;
/* After this message size switch to BTE protocols */
size_t btl_fma_limit;
size_t ugni_fma_limit;
/* Switch to put when trying to GET at or above this size */
size_t btl_get_limit;
size_t ugni_get_limit;
/* Switch to get when sending above this size */
size_t ugni_smsg_limit;
/* eager fragment list */
ompi_free_list_t ugni_frags_eager;
/* SMSG fragment list (unregistered) */
ompi_free_list_t ugni_frags_smsg;
/* RDMA fragment list */
ompi_free_list_t ugni_frags_rdma;
/* initial free list size */
/* RDMA/SMSG free list settings */
int ugni_free_list_num;
/* maximum free list size */
int ugni_free_list_max;
/* free list increment */
int ugni_free_list_inc;
/* eager free list settings */
int ugni_eager_num;
int ugni_eager_max;
int ugni_eager_inc;
/* number of times to retry a post */
int rdma_max_retries;
};
typedef struct mca_btl_ugni_component_t mca_btl_ugni_component_t;
/* Maximum number of outstanding eager messages */
int smsg_max_credits;
/* mailbox size (computed) */
int smsg_mbox_size;
} mca_btl_ugni_component_t;
int mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
ompi_common_ugni_device_t *device);
@ -232,13 +226,10 @@ mca_btl_ugni_alloc(struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
uint8_t order, size_t size, uint32_t flags);
struct mca_btl_ugni_reg_t {
typedef struct mca_btl_ugni_reg_t {
mca_mpool_base_registration_t base;
gni_mem_handle_t memory_hdl;
void *buffer;
size_t size;
};
typedef struct mca_btl_ugni_reg_t mca_btl_ugni_reg_t;
} mca_btl_ugni_reg_t;
/* Global structures */

184
ompi/mca/btl/ugni/btl_ugni_add_procs.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -23,7 +23,10 @@
#include "btl_ugni.h"
#include "btl_ugni_frag.h"
#include "btl_ugni_endpoint.h"
#include "btl_ugni_smsg.h"
static int
mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module);
int mca_btl_ugni_add_procs(struct mca_btl_base_module_t* btl,
size_t nprocs,
@ -45,27 +48,7 @@ int mca_btl_ugni_add_procs(struct mca_btl_base_module_t* btl,
return OMPI_ERR_OUT_OF_RESOURCE;
}
rc = ompi_free_list_init_new (&mca_btl_ugni_component.ugni_frags_eager,
sizeof (mca_btl_ugni_base_frag_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_ugni_base_frag_t),
sizeof (mca_btl_ugni_frag_hdr_t) + mca_btl_ugni_component.eager_limit,
opal_cache_line_size,
mca_btl_ugni_component.ugni_free_list_num,
mca_btl_ugni_component.ugni_free_list_max,
mca_btl_ugni_component.ugni_free_list_inc,
NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
rc = ompi_free_list_init_new (&mca_btl_ugni_component.ugni_frags_rdma,
sizeof (mca_btl_ugni_rdma_frag_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_ugni_rdma_frag_t),
0, opal_cache_line_size,
mca_btl_ugni_component.ugni_free_list_num,
mca_btl_ugni_component.ugni_free_list_max,
mca_btl_ugni_component.ugni_free_list_inc,
NULL);
rc = mca_btl_ugni_setup_mpools (ugni_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
@ -129,3 +112,158 @@ int mca_btl_ugni_del_procs (struct mca_btl_base_module_t *btl,
return OMPI_SUCCESS;
}
static int ugni_reg_mem (void *reg_data, void *base, size_t size,
mca_mpool_base_registration_t *reg)
{
mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *) reg_data;
mca_btl_ugni_reg_t *ugni_reg = (mca_btl_ugni_reg_t *) reg;
int rc;
rc = GNI_MemRegister (btl->device->dev_handle, (uint64_t) base,
size, NULL, GNI_MEM_READWRITE |
GNI_MEM_RELAXED_PI_ORDERING, -1,
&(ugni_reg->memory_hdl));
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS;
}
static int ugni_reg_smsg_mem (void *reg_data, void *base, size_t size,
mca_mpool_base_registration_t *reg)
{
mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *) reg_data;
mca_btl_ugni_reg_t *ugni_reg = (mca_btl_ugni_reg_t *) reg;
int rc;
rc = GNI_MemRegister (btl->device->dev_handle, (uint64_t)base,
size, btl->smsg_remote_cq, GNI_MEM_READWRITE,
-1, &(ugni_reg->memory_hdl));
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_SUCCESS;
}
static int
ugni_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
{
mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *) reg_data;
mca_btl_ugni_reg_t *ugni_reg = (mca_btl_ugni_reg_t *)reg;
int rc;
rc = GNI_MemDeregister (btl->device->dev_handle, &ugni_reg->memory_hdl);
if (GNI_RC_SUCCESS != rc) {
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
static int
mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module)
{
struct mca_mpool_base_resources_t mpool_resources;
int mbox_increment, rc;
size_t nprocs;
(void) ompi_proc_world (&nprocs);
rc = ompi_free_list_init_new (&mca_btl_ugni_component.ugni_frags_smsg,
sizeof (mca_btl_ugni_smsg_frag_t),
opal_cache_line_size, OBJ_CLASS(mca_btl_ugni_smsg_frag_t),
mca_btl_ugni_component.ugni_smsg_limit,
opal_cache_line_size,
mca_btl_ugni_component.ugni_free_list_num,
mca_btl_ugni_component.ugni_free_list_max,
mca_btl_ugni_component.ugni_free_list_inc,
NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
rc = ompi_free_list_init_new (&mca_btl_ugni_component.ugni_frags_rdma,
sizeof (mca_btl_ugni_rdma_frag_t), 8,
OBJ_CLASS(mca_btl_ugni_rdma_frag_t),
0, opal_cache_line_size,
mca_btl_ugni_component.ugni_free_list_num,
mca_btl_ugni_component.ugni_free_list_max,
mca_btl_ugni_component.ugni_free_list_inc,
NULL);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
mpool_resources.reg_data = (void *) ugni_module;
mpool_resources.sizeof_reg = sizeof (mca_btl_ugni_reg_t);
mpool_resources.register_mem = ugni_reg_mem;
mpool_resources.deregister_mem = ugni_dereg_mem;
ugni_module->super.btl_mpool =
mca_mpool_base_module_create("rdma", ugni_module->device,
&mpool_resources);
if (NULL == ugni_module->super.btl_mpool) {
BTL_ERROR(("error creating mpool"));
return OMPI_ERROR;
}
mpool_resources.register_mem = ugni_reg_smsg_mem;
ugni_module->smsg_mpool =
mca_mpool_base_module_create("rdma", ugni_module->device,
&mpool_resources);
OBJ_CONSTRUCT(&ugni_module->eager_frags_send, ompi_free_list_t);
rc = ompi_free_list_init_new (&ugni_module->eager_frags_send,
sizeof (mca_btl_ugni_eager_frag_t), 8,
OBJ_CLASS(mca_btl_ugni_eager_frag_t),
ugni_module->super.btl_eager_limit, 64,
mca_btl_ugni_component.ugni_eager_num,
mca_btl_ugni_component.ugni_eager_max,
mca_btl_ugni_component.ugni_eager_inc,
ugni_module->super.btl_mpool);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
OBJ_CONSTRUCT(&ugni_module->eager_frags_recv, ompi_free_list_t);
rc = ompi_free_list_init_new (&ugni_module->eager_frags_recv,
sizeof (mca_btl_ugni_eager_frag_t), 8,
OBJ_CLASS(mca_btl_ugni_eager_frag_t),
ugni_module->super.btl_eager_limit, 64,
mca_btl_ugni_component.ugni_eager_num,
mca_btl_ugni_component.ugni_eager_max,
mca_btl_ugni_component.ugni_eager_inc,
ugni_module->super.btl_mpool);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
OBJ_CONSTRUCT(&ugni_module->smsg_mboxes, ompi_free_list_t);
mbox_increment = nprocs;
/* limit mailbox allocations to at most 2MiB at a time */
if (nprocs * mca_btl_ugni_component.smsg_mbox_size > 2097152) {
mbox_increment = (int) (2097152.0 / (float)mca_btl_ugni_component.smsg_mbox_size);
}
rc = ompi_free_list_init_new (&ugni_module->smsg_mboxes,
sizeof (mca_btl_ugni_smsg_mbox_t), 8,
OBJ_CLASS(mca_btl_ugni_smsg_mbox_t),
mca_btl_ugni_component.smsg_mbox_size, 64,
0, nprocs, mbox_increment,
ugni_module->smsg_mpool);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
return OMPI_SUCCESS;
}

333
ompi/mca/btl/ugni/btl_ugni_component.c
Просмотреть файл

@ -11,18 +11,15 @@
*/
#include "btl_ugni.h"
#include "btl_ugni_endpoint.h"
#include "btl_ugni_frag.h"
#include "btl_ugni_rdma.h"
#include "btl_ugni_smsg.h"
#include "opal/include/opal/align.h"
#include "opal/mca/base/mca_base_param.h"
#include "opal/memoryhooks/memory.h"
#include "ompi/runtime/params.h"
int mca_btl_ugni_smsg_max_credits = 32;
int mca_btl_ugni_smsg_mbox_size;
static int btl_ugni_component_register(void);
static int btl_ugni_component_open(void);
static int btl_ugni_component_close(void);
@ -54,22 +51,12 @@ mca_btl_ugni_component_t mca_btl_ugni_component = {
}
};
static inline char *
mca_btl_ugni_param_register_string(const char *param_name,
const char *default_value)
{
char *param_value;
int id = mca_base_param_register_string("btl", "ugni", param_name, NULL,
default_value);
mca_base_param_lookup_string(id, &param_value);
return param_value;
}
static inline int
mca_btl_ugni_param_register_int (const char *param_name, int value)
mca_btl_ugni_param_register_int (const char *param_name, const char *help,
int value)
{
int id = mca_base_param_register_int("btl", "ugni", param_name, NULL, value);
mca_base_param_lookup_int(id, &value);
mca_base_param_reg_int(&mca_btl_ugni_component.super.btl_version,
param_name, help, false, false, value, &value);
return value;
}
@ -77,39 +64,62 @@ static int
btl_ugni_component_register(void)
{
mca_btl_ugni_component.ugni_free_list_num =
mca_btl_ugni_param_register_int("free_list_num", 8);
mca_btl_ugni_param_register_int("free_list_num", NULL, 8);
mca_btl_ugni_component.ugni_free_list_max =
mca_btl_ugni_param_register_int("free_list_max", -1);
mca_btl_ugni_param_register_int("free_list_max", NULL, -1);
mca_btl_ugni_component.ugni_free_list_inc =
mca_btl_ugni_param_register_int("free_list_inc", 64);
mca_btl_ugni_param_register_int("free_list_inc", NULL, 64);
mca_btl_ugni_component.ugni_eager_num =
mca_btl_ugni_param_register_int("eager_num", NULL, 16);
mca_btl_ugni_component.ugni_eager_max =
mca_btl_ugni_param_register_int("eager_max", NULL, 64);
mca_btl_ugni_component.ugni_eager_inc =
mca_btl_ugni_param_register_int("eager_inc", NULL, 16);
mca_btl_ugni_component.cq_size =
mca_btl_ugni_param_register_int("cq_size", 25000);
mca_btl_ugni_param_register_int("cq_size", NULL, 25000);
mca_btl_ugni_component.btl_fma_limit =
mca_btl_ugni_param_register_int("fma_limit", 4 * 1024);
/* SMSG limit. 0 - autoselect */
mca_btl_ugni_component.ugni_smsg_limit =
mca_btl_ugni_param_register_int("smsg_limit", "Maximum size message that "
"will be sent using the SMSG/MSGQ protocol "
"(0 - autoselect(default))", 0);
mca_btl_ugni_component.btl_get_limit =
mca_btl_ugni_param_register_int("get_limit", 8 * 1024);
mca_btl_ugni_component.smsg_max_credits =
mca_btl_ugni_param_register_int("smsg_max_credits", "Maximum number of "
"outstanding SMSG/MSGQ message (default 32)",
32);
mca_btl_ugni_component.ugni_fma_limit =
mca_btl_ugni_param_register_int("fma_limit", "Maximum size message that "
"will be sent using the FMA (Fast Memory "
"Access) protocol (default 4095)",
4 * 1024 - 1);
mca_btl_ugni_component.ugni_get_limit =
mca_btl_ugni_param_register_int("get_limit", NULL, 8 * 1024);
mca_btl_ugni_component.rdma_max_retries =
mca_btl_ugni_param_register_int("rdma_max_retries", 8);
mca_btl_ugni_param_register_int("rdma_max_retries", NULL, 8);
mca_btl_ugni_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
/* smsg threshold */
mca_btl_ugni_module.super.btl_eager_limit = 0; /* set dynamically in module_init */
mca_btl_ugni_module.super.btl_eager_limit = 8 * 1024;
mca_btl_ugni_module.super.btl_rndv_eager_limit = 8 * 1024;
mca_btl_ugni_module.super.btl_rdma_pipeline_frag_size = 2 * 1024 * 1024;
mca_btl_ugni_module.super.btl_max_send_size = 0; /* set this later */
mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = 0; /* set this later */
mca_btl_ugni_module.super.btl_max_send_size = 8 * 1024;
mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = 8 * 1024;
/* threshold for put */
mca_btl_ugni_module.super.btl_min_rdma_pipeline_size = 0;
mca_btl_ugni_module.super.btl_min_rdma_pipeline_size = 8 * 1024;
mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND |
MCA_BTL_FLAGS_RDMA |
MCA_BTL_FLAGS_RDMA_MATCHED;
MCA_BTL_FLAGS_RDMA_MATCHED |
MCA_BTL_FLAGS_SEND_INPLACE;
mca_btl_ugni_module.super.btl_bandwidth = 40000; /* Mbs */
mca_btl_ugni_module.super.btl_latency = 2; /* Microsecs */
@ -126,7 +136,7 @@ btl_ugni_component_open(void)
mca_btl_ugni_component.ugni_num_btls = 0;
mca_btl_ugni_component.modules = NULL;
OBJ_CONSTRUCT(&mca_btl_ugni_component.ugni_frags_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_ugni_component.ugni_frags_smsg, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_ugni_component.ugni_frags_rdma, ompi_free_list_t);
return OMPI_SUCCESS;
@ -140,7 +150,7 @@ btl_ugni_component_close(void)
{
ompi_common_ugni_fini ();
OBJ_DESTRUCT(&mca_btl_ugni_component.ugni_frags_eager);
OBJ_DESTRUCT(&mca_btl_ugni_component.ugni_frags_smsg);
OBJ_DESTRUCT(&mca_btl_ugni_component.ugni_frags_rdma);
return OMPI_SUCCESS;
@ -191,8 +201,8 @@ static int mca_btl_ugni_smsg_setup (void) {
/* calculate mailbox size */
tmp_smsg_attrib.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
tmp_smsg_attrib.msg_maxsize = mca_btl_ugni_component.eager_limit + sizeof (mca_btl_ugni_frag_hdr_t);
tmp_smsg_attrib.mbox_maxcredit = mca_btl_ugni_smsg_max_credits;
tmp_smsg_attrib.msg_maxsize = mca_btl_ugni_component.ugni_smsg_limit;
tmp_smsg_attrib.mbox_maxcredit = mca_btl_ugni_component.smsg_max_credits;
rc = GNI_SmsgBufferSizeNeeded (&tmp_smsg_attrib, &mbox_size);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
@ -200,9 +210,7 @@ static int mca_btl_ugni_smsg_setup (void) {
return ompi_common_rc_ugni_to_ompi (rc);
}
/* NTH: increase the mbox size to a multiple of 4096. ugni examples increase to a multiple of
64 bytes but we see hangs with anything less than 4096. */
mca_btl_ugni_smsg_mbox_size = OPAL_ALIGN(mbox_size, 4096, unsigned int);
mca_btl_ugni_component.smsg_mbox_size = OPAL_ALIGN(mbox_size, 64, unsigned int);
return OMPI_SUCCESS;
}
@ -218,6 +226,8 @@ mca_btl_ugni_component_init (int *num_btl_modules,
size_t nprocs;
int rc;
signal (SIGSEGV, SIG_DFL);
/* Initialize ugni library and create communication domain */
rc = ompi_common_ugni_init();
if (OMPI_SUCCESS != rc) {
@ -254,17 +264,24 @@ mca_btl_ugni_component_init (int *num_btl_modules,
(void) ompi_proc_world (&nprocs);
if (0 == mca_btl_ugni_component.eager_limit) {
/* auto-set the eager limit based on the number of ranks */
if (0 == mca_btl_ugni_component.ugni_smsg_limit) {
/* auto-set the smsg limit based on the number of ranks */
if (nprocs <= 1024) {
mca_btl_ugni_component.eager_limit = 1024;
mca_btl_ugni_component.ugni_smsg_limit = 1024;
} else if (nprocs <= 16384) {
mca_btl_ugni_component.eager_limit = 512;
mca_btl_ugni_component.ugni_smsg_limit = 512;
} else {
mca_btl_ugni_component.eager_limit = 256;
mca_btl_ugni_component.ugni_smsg_limit = 256;
}
}
mca_btl_ugni_component.smsg_max_data = mca_btl_ugni_component.ugni_smsg_limit -
sizeof (mca_btl_ugni_send_frag_hdr_t);
/* module settings */
mca_btl_ugni_module.super.btl_max_send_size = mca_btl_ugni_module.super.btl_eager_limit;
mca_btl_ugni_module.super.btl_rdma_pipeline_send_length = mca_btl_ugni_module.super.btl_eager_limit;
rc = mca_btl_ugni_smsg_setup ();
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return NULL;
@ -291,57 +308,133 @@ mca_btl_ugni_component_init (int *num_btl_modules,
return base_modules;
}
static inline void mca_btl_ugni_callback_reverse_get (mca_btl_base_module_t *btl,
mca_btl_base_endpoint_t *ep,
mca_btl_base_descriptor_t *des,
int rc)
static void mca_btl_ugni_callback_reverse_get (ompi_common_ugni_post_desc_t *desc, int rc)
{
mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) des;
mca_btl_ugni_base_frag_t *frag = MCA_BTL_UGNI_DESC_TO_FRAG(desc);
BTL_VERBOSE(("reverse get (put) for rem_ctx %p complete", des->des_cbdata));
BTL_VERBOSE(("reverse get (put) for rem_ctx %p complete", frag->hdr.rdma.ctx));
/* tell peer the put is complete */
rc = GNI_SmsgSendWTag (frag->endpoint->common->ep_handle, &des->des_cbdata, sizeof (void *),
NULL, 0, -1, MCA_BTL_UGNI_TAG_PUT_COMPLETE);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
/* turn off btl ownership for now */
des->des_flags &= ~MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
opal_list_append (&ugni_module->failed_frags, (opal_list_item_t *) des);
} else {
des->des_flags |= MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
rc = ompi_mca_btl_ugni_smsg_send (frag, false, &frag->hdr.rdma, sizeof (frag->hdr.rdma),
NULL, 0, MCA_BTL_UGNI_TAG_PUT_COMPLETE);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
opal_list_append (&frag->endpoint->btl->failed_frags, (opal_list_item_t *) frag);
}
}
static inline int mca_btl_ugni_start_progress_reverse_get (mca_btl_base_endpoint_t *ep,
mca_btl_base_segment_t *segments,
void *rem_ctx)
static void mca_btl_ugni_callback_eager_get (ompi_common_ugni_post_desc_t *desc, int rc)
{
mca_btl_ugni_base_frag_t *frag = MCA_BTL_UGNI_DESC_TO_FRAG(desc);
mca_btl_active_message_callback_t *reg;
BTL_VERBOSE(("eager get for rem_ctx %p complete", frag->hdr.eager.ctx));
/* the frag is already set up for the send callback */
frag->segments[0].seg_len = frag->hdr.eager.len;
reg = mca_btl_base_active_message_trigger + frag->hdr.eager.tag;
reg->cbfunc(&frag->endpoint->btl->super, frag->hdr.eager.tag, &(frag->base), reg->cbdata);
/* tell peer the get is complete */
rc = ompi_mca_btl_ugni_smsg_send (frag, false, &frag->hdr.eager, sizeof (frag->hdr.eager),
NULL, 0, MCA_BTL_UGNI_TAG_GET_COMPLETE);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
opal_list_append (&frag->endpoint->btl->failed_frags, (opal_list_item_t *) frag);
}
}
static inline int mca_btl_ugni_start_reverse_get (mca_btl_base_endpoint_t *ep,
mca_btl_ugni_rdma_frag_hdr_t hdr)
{
mca_btl_ugni_base_frag_t *frag;
int rc;
BTL_VERBOSE(("starting reverse get (put) for remote ctx: %p", rem_ctx));
BTL_VERBOSE(("starting reverse get (put) for remote ctx: %p", hdr.ctx));
MCA_BTL_UGNI_FRAG_ALLOC_RDMA(ep->btl, frag, rc);
rc = MCA_BTL_UGNI_FRAG_ALLOC_RDMA(ep, frag);
if (OPAL_UNLIKELY(NULL == frag)) {
BTL_ERROR(("error allocating rdma frag for reverse get"));
return rc;
}
frag->base.des_cbfunc = mca_btl_ugni_callback_reverse_get;
frag->base.des_cbdata = rem_ctx;
frag->endpoint = ep;
frag->hdr.rdma = hdr;
memmove (&frag->segments, segments, 2 * sizeof (segments[0]));
frag->base.des_cbfunc = NULL;
frag->base.des_flags = MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
frag->segments[0] = hdr.src_seg;
frag->base.des_src = frag->segments;
frag->base.des_src_cnt = 1;
frag->segments[1] = hdr.dst_seg;
frag->base.des_dst = frag->segments + 1;
frag->base.des_dst_cnt = 1;
rc = mca_btl_ugni_put (&ep->btl->super, ep, &frag->base);
assert (OMPI_SUCCESS == rc);
frag->post_desc.cbfunc = mca_btl_ugni_callback_reverse_get;
return rc;
}
static inline int mca_btl_ugni_start_eager_get (mca_btl_base_endpoint_t *ep,
mca_btl_ugni_eager_frag_hdr_t hdr);
static void mca_btl_ugni_callback_eager_get_retry (ompi_common_ugni_post_desc_t *desc, int rc)
{
mca_btl_ugni_base_frag_t *frag = MCA_BTL_UGNI_DESC_TO_FRAG(desc);
(void) mca_btl_ugni_start_eager_get(frag->endpoint, frag->hdr.eager);
mca_btl_ugni_frag_return (frag);
}
static inline int mca_btl_ugni_start_eager_get (mca_btl_base_endpoint_t *ep,
mca_btl_ugni_eager_frag_hdr_t hdr)
{
mca_btl_ugni_base_frag_t *frag;
int rc;
BTL_VERBOSE(("starting eager get for remote ctx: %p", hdr.ctx));
rc = MCA_BTL_UGNI_FRAG_ALLOC_EAGER_RECV(ep, frag);
if (OPAL_UNLIKELY(NULL == frag)) {
(void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(ep, frag);
assert (NULL != frag);
frag->hdr.eager = hdr;
frag->post_desc.cbfunc = mca_btl_ugni_callback_eager_get_retry;
opal_list_append (&ep->btl->failed_frags, (opal_list_item_t *) frag);
return rc;
}
frag->hdr.eager = hdr;
frag->base.des_cbfunc = NULL;
frag->base.des_flags = MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
frag->base.des_dst = frag->segments;
frag->base.des_dst_cnt = 1;
frag->segments[1] = hdr.src_seg;
frag->base.des_src = frag->segments + 1;
frag->base.des_src_cnt = 1;
/* increase size to a multiple of 4 bytes (required for get) */
frag->segments[0].seg_len = (hdr.len + 3) & ~3;
frag->segments[1].seg_len = (hdr.len + 3) & ~3;
if (frag->segments[0].seg_len <= mca_btl_ugni_component.ugni_fma_limit) {
rc = mca_btl_ugni_post_fma (frag, GNI_POST_FMA_GET, frag->base.des_dst, frag->base.des_src);
} else {
rc = mca_btl_ugni_post_bte (frag, GNI_POST_RDMA_GET, frag->base.des_dst, frag->base.des_src);
}
assert (OMPI_SUCCESS == rc);
frag->post_desc.cbfunc = mca_btl_ugni_callback_eager_get;
return rc;
}
@ -350,11 +443,10 @@ mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
{
mca_btl_active_message_callback_t *reg;
mca_btl_ugni_base_frag_t frag;
mca_btl_base_segment_t *segments;
mca_btl_ugni_frag_hdr_t *hdr;
bool disconnect = false;
uintptr_t data_ptr;
gni_return_t rc;
int count = 0;
int rc;
/* per uGNI documentation we loop until the mailbox is empty */
do {
@ -363,7 +455,8 @@ mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
rc = GNI_SmsgGetNextWTag (ep->common->ep_handle, (void **) &data_ptr, &tag);
if (GNI_RC_NOT_DONE == rc) {
BTL_VERBOSE(("no smsg message waiting. rc = %d", rc));
break;
return count;
}
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
@ -374,6 +467,7 @@ mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
if (OPAL_UNLIKELY(0 == data_ptr)) {
BTL_ERROR(("null data ptr!"));
assert (0);
return OMPI_ERROR;
}
@ -383,42 +477,44 @@ mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
switch (tag) {
case MCA_BTL_UGNI_TAG_SEND:
hdr = (mca_btl_ugni_frag_hdr_t *) data_ptr;
frag.hdr.send = ((mca_btl_ugni_send_frag_hdr_t *) data_ptr)[0];
BTL_VERBOSE(("received smsg fragment. hdr = {len = %u, tag = %d}",
(unsigned int) hdr->len, hdr->tag));
(unsigned int) frag.hdr.send.len, frag.hdr.send.tag));
reg = mca_btl_base_active_message_trigger + hdr->tag;
reg = mca_btl_base_active_message_trigger + frag.hdr.send.tag;
frag.base.des_dst = frag.segments;
frag.base.des_dst_cnt = 1;
frag.segments[0].seg_addr.pval = (void *)(data_ptr + sizeof (*hdr));
frag.segments[0].seg_len = hdr->len;
frag.segments[0].seg_addr.pval = (void *)((uintptr_t)data_ptr + sizeof (mca_btl_ugni_send_frag_hdr_t));
frag.segments[0].seg_len = frag.hdr.send.len;
reg->cbfunc(&ep->btl->super, hdr->tag, &(frag.base), reg->cbdata);
reg->cbfunc(&ep->btl->super, frag.hdr.send.tag, &(frag.base), reg->cbdata);
break;
case MCA_BTL_UGNI_TAG_PUT_INIT:
frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_start_reverse_get (ep, frag.hdr.rdma);
break;
case MCA_BTL_UGNI_TAG_PUT_COMPLETE:
frag.hdr.rdma = ((mca_btl_ugni_rdma_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_post_frag_complete (frag.hdr.rdma.ctx, OMPI_SUCCESS);
break;
case MCA_BTL_UGNI_TAG_GET_INIT:
frag.hdr.eager = ((mca_btl_ugni_eager_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_start_eager_get (ep, frag.hdr.eager);
break;
case MCA_BTL_UGNI_TAG_GET_COMPLETE:
frag.hdr.eager = ((mca_btl_ugni_eager_frag_hdr_t *) data_ptr)[0];
mca_btl_ugni_post_frag_complete (frag.hdr.eager.ctx, OMPI_SUCCESS);
break;
case MCA_BTL_UGNI_TAG_DISCONNECT:
/* remote endpoint has disconnected */
rc = GNI_SmsgRelease (ep->common->ep_handle);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
BTL_ERROR(("Smsg release failed!"));
return OMPI_ERROR;
}
mca_btl_ugni_ep_disconnect (ep, false);
return count;
case MCA_BTL_UGNI_TAG_PUT_INIT:
segments = (mca_btl_base_segment_t *) data_ptr;
mca_btl_ugni_start_progress_reverse_get (ep, segments,
((void **)(segments + 2))[0]);
break;
case MCA_BTL_UGNI_TAG_PUT_COMPLETE:
mca_btl_ugni_post_frag_complete (((void **)data_ptr)[0], OMPI_SUCCESS);
disconnect = true;
break;
default:
BTL_ERROR(("unknown tag %d\n", tag));
@ -430,9 +526,11 @@ mca_btl_ugni_smsg_process (mca_btl_base_endpoint_t *ep)
BTL_ERROR(("Smsg release failed!"));
return OMPI_ERROR;
}
} while (1);
} while (!disconnect);
/* disconnect if we get here */
mca_btl_ugni_ep_disconnect (ep, false);
/* finished processing events */
return count;
}
@ -440,17 +538,16 @@ static inline int
mca_btl_ugni_progress_datagram (mca_btl_ugni_module_t *btl)
{
uint32_t remote_addr, remote_id;
uint64_t datagram_id;
mca_btl_base_endpoint_t *ep;
gni_ep_handle_t handle;
gni_post_state_t post_state;
int rc, count;
gni_ep_handle_t handle;
uint64_t datagram_id;
gni_return_t grc;
int count = 0;
count = 0;
post_state = GNI_POST_PENDING;
rc = GNI_PostDataProbeById (btl->device->dev_handle, &datagram_id);
if (OPAL_LIKELY(GNI_RC_SUCCESS != rc)) {
/* check for datagram completion */
grc = GNI_PostDataProbeById (btl->device->dev_handle, &datagram_id);
if (OPAL_LIKELY(GNI_RC_SUCCESS != grc)) {
return 0;
}
@ -463,11 +560,11 @@ mca_btl_ugni_progress_datagram (mca_btl_ugni_module_t *btl)
}
/* wait for the incoming datagram to complete (in case it isn't) */
rc = GNI_EpPostDataWaitById (handle, datagram_id, -1, &post_state,
grc = GNI_EpPostDataWaitById (handle, datagram_id, -1, &post_state,
&remote_addr, &remote_id);
if (GNI_RC_SUCCESS != rc) {
BTL_ERROR(("GNI_EpPostDataWaitById failed with rc = %d", rc));
return ompi_common_rc_ugni_to_ompi (rc);
if (GNI_RC_SUCCESS != grc) {
BTL_ERROR(("GNI_EpPostDataWaitById failed with rc = %d", grc));
return ompi_common_rc_ugni_to_ompi (grc);
}
BTL_VERBOSE(("got a datagram completion: id = %" PRIx64 ", state = %d, "
@ -480,10 +577,9 @@ mca_btl_ugni_progress_datagram (mca_btl_ugni_module_t *btl)
/* NTH: TODO -- error handling */
(void) mca_btl_ugni_ep_connect_progress (ep);
if (ep->smsgs_waiting && OMPI_COMMON_UGNI_CONNECTED == MCA_BTL_UGNI_EP_STATE(ep)) {
if (OMPI_COMMON_UGNI_CONNECTED == MCA_BTL_UGNI_EP_STATE(ep)) {
/* process messages waiting in the endpoint's smsg mailbox */
while ((rc = mca_btl_ugni_smsg_process (ep) > 0)) count += rc;
ep->smsgs_waiting = false;
count = mca_btl_ugni_smsg_process (ep);
}
OPAL_THREAD_UNLOCK(&ep->common->lock);
@ -507,7 +603,7 @@ mca_btl_ugni_handle_smsg_overrun (mca_btl_ugni_module_t *btl)
"processing message backlog..."));
/* we don't know which endpoint lost an smsg completion. clear the
smsg cq and check all mailboxes */
smsg remote cq and check all mailboxes */
/* clear out remote cq */
do {
@ -573,9 +669,6 @@ mca_btl_ugni_progress_smsg (mca_btl_ugni_module_t *btl)
/* due to the nature of datagrams we may get a smsg completion before
we get mailbox info from the peer */
BTL_VERBOSE(("event occurred on an unconnected endpoint! ep state = %d", MCA_BTL_UGNI_EP_STATE(ep)));
/* flag the endpoint as having messages waiting */
ep->smsgs_waiting = true;
return 0;
}
@ -595,7 +688,9 @@ mca_btl_ugni_retry_failed (mca_btl_ugni_module_t *btl)
opal_list_item_t *item;
while (count-- && NULL != (item = opal_list_remove_first (&btl->failed_frags))) {
mca_btl_ugni_post_frag_complete ((void *) item, OMPI_SUCCESS);
mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) item;
frag->post_desc.cbfunc (&frag->post_desc, OMPI_SUCCESS);
}
return 0;

24
ompi/mca/btl/ugni/btl_ugni_endpoint.c
Просмотреть файл

@ -13,6 +13,7 @@
#include "btl_ugni.h"
#include "btl_ugni_endpoint.h"
#include "btl_ugni_frag.h"
#include "btl_ugni_smsg.h"
static void mca_btl_ugni_ep_construct (mca_btl_base_endpoint_t *ep);
static void mca_btl_ugni_ep_destruct (mca_btl_base_endpoint_t *ep);
@ -24,7 +25,6 @@ static void mca_btl_ugni_ep_construct (mca_btl_base_endpoint_t *ep)
{
OBJ_CONSTRUCT(&ep->pending_list, opal_list_t);
OBJ_CONSTRUCT(&ep->pending_smsg_sends, opal_list_t);
ep->smsgs_waiting = false;
}
static void mca_btl_ugni_ep_destruct (mca_btl_base_endpoint_t *ep)
@ -37,16 +37,13 @@ static void mca_btl_ugni_smsg_mbox_construct (mca_btl_ugni_smsg_mbox_t *mbox) {
struct mca_btl_ugni_reg_t *reg =
(struct mca_btl_ugni_reg_t *) mbox->super.registration;
mbox->buffer = mbox->super.ptr;
/* initialize mailbox attributes */
mbox->smsg_attrib.msg_type = GNI_SMSG_TYPE_MBOX_AUTO_RETRANSMIT;
mbox->smsg_attrib.msg_maxsize = mca_btl_ugni_component.eager_limit +
sizeof (mca_btl_ugni_frag_hdr_t);
mbox->smsg_attrib.mbox_maxcredit = mca_btl_ugni_smsg_max_credits;
mbox->smsg_attrib.mbox_offset = 0; /* autoselect */
mbox->smsg_attrib.msg_buffer = mbox->buffer;
mbox->smsg_attrib.buff_size = mca_btl_ugni_smsg_mbox_size;
mbox->smsg_attrib.msg_maxsize = mca_btl_ugni_component.ugni_smsg_limit;
mbox->smsg_attrib.mbox_maxcredit = mca_btl_ugni_component.smsg_max_credits;
mbox->smsg_attrib.mbox_offset = (uintptr_t) mbox->super.ptr - (uintptr_t) reg->base.alloc_base;
mbox->smsg_attrib.msg_buffer = reg->base.alloc_base;
mbox->smsg_attrib.buff_size = mca_btl_ugni_component.smsg_mbox_size;
mbox->smsg_attrib.mem_hndl = reg->memory_hdl;
}
@ -66,13 +63,12 @@ static inline int mca_btl_ugni_ep_smsg_get_mbox (mca_btl_base_endpoint_t *ep) {
ep->mailbox = (mca_btl_ugni_smsg_mbox_t *) mbox;
/* per ugni spec we need to zero mailbox data before connecting */
memset (ep->mailbox->buffer, 0, mca_btl_ugni_smsg_mbox_size);
memset ((char *)ep->mailbox->smsg_attrib.msg_buffer + ep->mailbox->smsg_attrib.mbox_offset, 0,
ep->mailbox->smsg_attrib.buff_size);
return rc;
}
int mca_btl_ugni_ep_disconnect (mca_btl_base_endpoint_t *ep, bool send_disconnect) {
uint32_t msg_id = ORTE_PROC_MY_NAME->vpid;
char msg;
int rc;
OPAL_THREAD_LOCK(&ep->common->lock);
@ -84,7 +80,7 @@ int mca_btl_ugni_ep_disconnect (mca_btl_base_endpoint_t *ep, bool send_disconnec
}
if (OMPI_COMMON_UGNI_CONNECTED == MCA_BTL_UGNI_EP_STATE(ep) && send_disconnect) {
rc = GNI_SmsgSendWTag (ep->common->ep_handle, &msg, 1, NULL, 0, msg_id,
rc = GNI_SmsgSendWTag (ep->common->ep_handle, NULL, 0, NULL, 0, -1,
MCA_BTL_UGNI_TAG_DISCONNECT);
if (GNI_RC_SUCCESS != rc) {
BTL_VERBOSE(("btl/ugni could not send close message"));
@ -173,7 +169,7 @@ static inline int mca_btl_ugni_ep_connect_finish (mca_btl_base_endpoint_t *ep) {
while (NULL != (item = opal_list_remove_first (&ep->pending_list))) {
mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) item;
(void) mca_btl_ugni_send (&ep->btl->super, ep, &frag->base, frag->tag);
(void) mca_btl_ugni_send (&ep->btl->super, ep, &frag->base, frag->hdr.send.tag);
}
return OMPI_SUCCESS;

19
ompi/mca/btl/ugni/btl_ugni_endpoint.h
Просмотреть файл

@ -15,17 +15,7 @@
#include "btl_ugni.h"
struct mca_btl_ugni_smsg_mbox_t {
ompi_free_list_item_t super;
void *buffer;
gni_smsg_attr_t smsg_attrib;
};
typedef struct mca_btl_ugni_smsg_mbox_t mca_btl_ugni_smsg_mbox_t;
OBJ_CLASS_DECLARATION(mca_btl_ugni_smsg_mbox_t);
struct mca_btl_base_endpoint_t {
typedef struct mca_btl_base_endpoint_t {
opal_object_t super;
ompi_common_ugni_endpoint_t *common;
@ -38,15 +28,12 @@ struct mca_btl_base_endpoint_t {
opal_list_t pending_list;
opal_list_t pending_smsg_sends;
} mca_btl_base_endpoint_t;
/* true if a frag was received before the connection was complete */
bool smsgs_waiting;
};
typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
OBJ_CLASS_DECLARATION(mca_btl_base_endpoint_t);
#define MCA_BTL_UGNI_EP_STATE(ep) ((ep)->common->state)
OBJ_CLASS_DECLARATION(mca_btl_base_endpoint_t);
int mca_btl_ugni_ep_connect_progress (mca_btl_base_endpoint_t *ep);
int mca_btl_ugni_ep_disconnect (mca_btl_base_endpoint_t *ep, bool send_disconnect);

30
ompi/mca/btl/ugni/btl_ugni_frag.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -13,18 +13,17 @@
#include "btl_ugni.h"
#include "btl_ugni_frag.h"
static inline void mca_btl_ugni_frag_constructor (mca_btl_ugni_base_frag_t *frag)
static inline void mca_btl_ugni_smsg_frag_constructor (mca_btl_ugni_base_frag_t *frag)
{
/* send memory does not need to be registered so we do not need a mpool */
memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
frag->hdr = (mca_btl_ugni_frag_hdr_t *) calloc (1, sizeof (mca_btl_ugni_frag_hdr_t) + mca_btl_ugni_component.eager_limit);
frag->segments[0].seg_addr.pval = (void *) (frag->hdr + 1);
frag->buffer = frag->segments[0].seg_addr.pval = calloc (1, mca_btl_ugni_component.smsg_max_data);
}
static inline void mca_btl_ugni_frag_destructor (mca_btl_ugni_base_frag_t *frag)
{
if (NULL != frag->hdr) {
free (frag->hdr);
if (NULL != frag->buffer) {
free (frag->buffer);
}
}
@ -34,8 +33,21 @@ static inline void mca_btl_ugni_rdma_frag_constructor (mca_btl_ugni_base_frag_t
memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
}
OBJ_CLASS_INSTANCE(mca_btl_ugni_base_frag_t, mca_btl_base_descriptor_t,
mca_btl_ugni_frag_constructor, mca_btl_ugni_frag_destructor);
static inline void mca_btl_ugni_eager_frag_constructor (mca_btl_ugni_base_frag_t *frag)
{
struct mca_btl_ugni_reg_t *reg =
(struct mca_btl_ugni_reg_t *) frag->base.super.registration;
memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
frag->segments[0].seg_addr.pval = frag->base.super.ptr;
memmove (frag->segments[0].seg_key.key64, &reg->memory_hdl, sizeof (reg->memory_hdl));
}
OBJ_CLASS_INSTANCE(mca_btl_ugni_smsg_frag_t, mca_btl_base_descriptor_t,
mca_btl_ugni_smsg_frag_constructor, mca_btl_ugni_frag_destructor);
OBJ_CLASS_INSTANCE(mca_btl_ugni_rdma_frag_t, mca_btl_base_descriptor_t,
mca_btl_ugni_rdma_frag_constructor, NULL);
mca_btl_ugni_rdma_frag_constructor, mca_btl_ugni_frag_destructor);
OBJ_CLASS_INSTANCE(mca_btl_ugni_eager_frag_t, mca_btl_base_descriptor_t,
mca_btl_ugni_eager_frag_constructor, mca_btl_ugni_frag_destructor);

102
ompi/mca/btl/ugni/btl_ugni_frag.h
Просмотреть файл

@ -16,60 +16,88 @@
#include "btl_ugni.h"
#include "btl_ugni_endpoint.h"
struct mca_btl_ugni_frag_hdr_t {
typedef struct mca_btl_ugni_send_frag_hdr_t {
size_t len;
mca_btl_base_tag_t tag;
};
typedef struct mca_btl_ugni_frag_hdr_t mca_btl_ugni_frag_hdr_t;
} mca_btl_ugni_send_frag_hdr_t;
struct mca_btl_ugni_base_frag_t {
typedef struct mca_btl_ugni_rdma_frag_hdr_t {
mca_btl_base_segment_t src_seg;
mca_btl_base_segment_t dst_seg;
void *ctx;
} mca_btl_ugni_rdma_frag_hdr_t;
typedef struct mca_btl_ugni_eager_frag_hdr_t {
size_t len;
mca_btl_base_tag_t tag;
mca_btl_base_segment_t src_seg;
void *ctx;
} mca_btl_ugni_eager_frag_hdr_t;
typedef union mca_btl_ugni_frag_hdr_t {
mca_btl_ugni_send_frag_hdr_t send;
mca_btl_ugni_rdma_frag_hdr_t rdma;
mca_btl_ugni_eager_frag_hdr_t eager;
} mca_btl_ugni_frag_hdr_t;
typedef struct mca_btl_ugni_base_frag_t {
mca_btl_base_descriptor_t base;
mca_btl_base_segment_t segments[2];
mca_btl_ugni_frag_hdr_t *hdr;
mca_btl_base_tag_t tag;
mca_btl_ugni_frag_hdr_t hdr;
ompi_common_ugni_post_desc_t post_desc;
mca_btl_base_endpoint_t *endpoint;
mca_btl_ugni_reg_t *registration;
ompi_free_list_t *my_list;
mca_btl_ugni_module_t *btl;
int tries;
void *buffer;
uint32_t msg_id;
};
} mca_btl_ugni_base_frag_t;
typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_base_frag_t;
typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_smsg_frag_t;
typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_rdma_frag_t;
typedef struct mca_btl_ugni_base_frag_t mca_btl_ugni_eager_frag_t;
#define MCA_BTL_UGNI_DESC_TO_FRAG(desc) ((mca_btl_ugni_base_frag_t *)((uintptr_t) (desc) - offsetof (mca_btl_ugni_base_frag_t, post_desc)))
#define MCA_BTL_UGNI_DESC_TO_FRAG(desc) \
((mca_btl_ugni_base_frag_t *)((uintptr_t) (desc) - offsetof (mca_btl_ugni_base_frag_t, post_desc)))
OBJ_CLASS_DECLARATION(mca_btl_ugni_base_frag_t);
OBJ_CLASS_DECLARATION(mca_btl_ugni_smsg_frag_t);
OBJ_CLASS_DECLARATION(mca_btl_ugni_rdma_frag_t);
OBJ_CLASS_DECLARATION(mca_btl_ugni_eager_frag_t);
#define MCA_BTL_UGNI_FRAG_ALLOC_EAGER(module, frag, rc) \
do { \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET(&mca_btl_ugni_component.ugni_frags_eager, item, rc); \
frag = (mca_btl_ugni_base_frag_t *) item; \
frag->my_list = &mca_btl_ugni_component.ugni_frags_eager; \
frag->btl = (module); \
} while (0)
static inline int mca_btl_ugni_frag_alloc (mca_btl_base_endpoint_t *ep,
ompi_free_list_t *list,
mca_btl_ugni_base_frag_t **frag)
{
ompi_free_list_item_t *item = NULL;
int rc;
#define MCA_BTL_UGNI_FRAG_ALLOC_RDMA(module, frag, rc) \
do { \
ompi_free_list_item_t *item; \
OMPI_FREE_LIST_GET(&mca_btl_ugni_component.ugni_frags_rdma, item, rc); \
frag = (mca_btl_ugni_base_frag_t *) item; \
frag->my_list = &mca_btl_ugni_component.ugni_frags_rdma; \
frag->btl = (module); \
} while (0)
OMPI_FREE_LIST_GET(list, item, rc);
*frag = (mca_btl_ugni_base_frag_t *) item;
if (OPAL_LIKELY(NULL != item)) {
(*frag)->my_list = list;
(*frag)->endpoint = ep;
}
#define MCA_BTL_UGNI_FRAG_RETURN(frag) \
do { \
if (OPAL_UNLIKELY(NULL != (frag)->registration)) { \
(frag)->btl->super.btl_mpool->mpool_deregister((frag)->btl->super.btl_mpool, \
&(frag)->registration->base); \
(frag)->registration = NULL; \
} \
OMPI_FREE_LIST_RETURN((frag)->my_list, (ompi_free_list_item_t *)(frag)); \
} while (0);
return rc;
}
static inline void mca_btl_ugni_frag_return (mca_btl_ugni_base_frag_t *frag)
{
if (frag->registration) {
frag->endpoint->btl->super.btl_mpool->mpool_deregister(frag->endpoint->btl->super.btl_mpool,
&frag->registration->base);
frag->registration = NULL;
}
OMPI_FREE_LIST_RETURN(frag->my_list, (ompi_free_list_item_t *) frag);
}
#define MCA_BTL_UGNI_FRAG_ALLOC_SMSG(ep, frag) \
mca_btl_ugni_frag_alloc((ep), &mca_btl_ugni_component.ugni_frags_smsg, &(frag))
#define MCA_BTL_UGNI_FRAG_ALLOC_RDMA(ep, frag) \
mca_btl_ugni_frag_alloc((ep), &mca_btl_ugni_component.ugni_frags_rdma, &(frag))
#define MCA_BTL_UGNI_FRAG_ALLOC_EAGER_SEND(ep, frag) \
mca_btl_ugni_frag_alloc((ep), &(ep)->btl->eager_frags_send, &(frag))
#define MCA_BTL_UGNI_FRAG_ALLOC_EAGER_RECV(ep, frag) \
mca_btl_ugni_frag_alloc((ep), &(ep)->btl->eager_frags_recv, &(frag))
#endif /* MCA_BTL_UGNI_FRAG_H */

52
ompi/mca/btl/ugni/btl_ugni_get.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -11,6 +11,20 @@
*/
#include "btl_ugni_rdma.h"
#include "btl_ugni_smsg.h"
static inline int mca_btl_ugni_init_reverse_get (struct mca_btl_base_module_t *btl,
mca_btl_ugni_base_frag_t *frag) {
/* off alignment/off size. switch to put */
frag->hdr.rdma.src_seg = frag->base.des_src[0];
frag->hdr.rdma.dst_seg = frag->base.des_dst[0];
frag->hdr.rdma.ctx = (void *) &frag->post_desc;
/* send the fragment header using smsg. ignore local completion */
return ompi_mca_btl_ugni_smsg_send (frag, true, &frag->hdr.rdma,
sizeof (frag->hdr.rdma), NULL, 0,
MCA_BTL_UGNI_TAG_PUT_INIT);
}
/**
* Initiate a get operation.
@ -23,43 +37,29 @@ int mca_btl_ugni_get (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
struct mca_btl_base_descriptor_t *des) {
mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) des;
gni_mem_handle_t lcl_hdl, rem_hdl;
void *lcl_buffer, *rem_buffer;
size_t size;
size_t size = des->des_src->seg_len;
bool check;
int rc;
BTL_VERBOSE(("Using RDMA Get"));
BTL_VERBOSE(("Using RDMA/FMA Get"));
/* Check if endpoint is connected */
rc = mca_btl_ugni_check_endpoint_state(endpoint);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc))
/* we should already be connected by this point (we got a smsg send) */
/* Ack! we should already be connected by this point (we got a smsg msg) */
return rc;
/* Get remote memory handle */
rem_buffer = (void *)(des->des_src->seg_addr.pval);
size = des->des_src->seg_len;
memcpy (&rem_hdl, (void *) des->des_src->seg_key.key64, sizeof (rem_hdl));
/* Check if the get is aligned/sized on a multiple of 4 */
check = !!((des->des_src->seg_addr.lval | des->des_dst->seg_addr.lval | size) & 3);
/* Get local memory handle */
lcl_buffer = (void *)(des->des_dst->seg_addr.pval);
memcpy (&lcl_hdl, (void *) des->des_dst->seg_key.key64, sizeof (lcl_hdl));
if (OPAL_UNLIKELY(((uintptr_t)rem_buffer & 0x3) || ((uintptr_t)lcl_buffer & 0x3) ||
size & 0x3 || size > mca_btl_ugni_component.btl_get_limit)) {
if (OPAL_UNLIKELY(check || size > mca_btl_ugni_component.ugni_get_limit)) {
/* switch to put */
return mca_btl_ugni_start_reverse_get (btl, frag);
return mca_btl_ugni_init_reverse_get (btl, frag);
}
frag->tries = 0;
if (size < mca_btl_ugni_component.btl_fma_limit) {
rc = post_fma_descriptor (frag, GNI_POST_FMA_GET, endpoint, size,
lcl_buffer, lcl_hdl, rem_buffer, rem_hdl);
} else {
rc = post_bte_descriptor (frag, GNI_POST_RDMA_GET, endpoint, size,
lcl_buffer, lcl_hdl, rem_buffer, rem_hdl);
if (size <= mca_btl_ugni_component.ugni_fma_limit) {
return mca_btl_ugni_post_fma (frag, GNI_POST_FMA_GET, des->des_dst, des->des_src);
}
return rc;
return mca_btl_ugni_post_bte (frag, GNI_POST_RDMA_GET, des->des_dst, des->des_src);
}

184
ompi/mca/btl/ugni/btl_ugni_module.c
Просмотреть файл

@ -25,6 +25,7 @@
#include "btl_ugni.h"
#include "btl_ugni_frag.h"
#include "btl_ugni_endpoint.h"
#include "btl_ugni_smsg.h"
#include <unistd.h>
#include <sys/types.h>
@ -80,7 +81,7 @@ mca_btl_ugni_module_t mca_btl_ugni_module = {
mca_btl_ugni_prepare_src,
mca_btl_ugni_prepare_dst,
mca_btl_ugni_send,
NULL, /* sendi */
mca_btl_ugni_sendi,
mca_btl_ugni_put,
mca_btl_ugni_get,
NULL, /* mca_btl_base_dump, */
@ -90,116 +91,6 @@ mca_btl_ugni_module_t mca_btl_ugni_module = {
}
};
static int ugni_reg_mem (void *reg_data, void *base, size_t size,
mca_mpool_base_registration_t *reg)
{
mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *) reg_data;
mca_btl_ugni_reg_t *ugni_reg = (mca_btl_ugni_reg_t *) reg;
int rc;
rc = GNI_MemRegister (btl->device->dev_handle, (uint64_t)base,
size, NULL, GNI_MEM_READWRITE, -1,
&(ugni_reg->memory_hdl));
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
ugni_reg->buffer = base;
ugni_reg->size = size;
return OMPI_SUCCESS;
}
static int ugni_reg_smsg_mem (void *reg_data, void *base, size_t size,
mca_mpool_base_registration_t *reg)
{
mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *) reg_data;
mca_btl_ugni_reg_t *ugni_reg = (mca_btl_ugni_reg_t *) reg;
int rc;
rc = GNI_MemRegister (btl->device->dev_handle, (uint64_t)base,
size, btl->smsg_remote_cq, GNI_MEM_READWRITE |
GNI_MEM_USE_GART, -1, &(ugni_reg->memory_hdl));
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
ugni_reg->buffer = base;
ugni_reg->size = size;
return OMPI_SUCCESS;
}
static int
ugni_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
{
mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *) reg_data;
mca_btl_ugni_reg_t *ugni_reg = (mca_btl_ugni_reg_t *)reg;
int rc;
rc = GNI_MemDeregister (btl->device->dev_handle, &ugni_reg->memory_hdl);
if (GNI_RC_SUCCESS != rc) {
return OMPI_ERROR;
}
ugni_reg->buffer = NULL;
ugni_reg->size = 0;
return OMPI_SUCCESS;
}
static int
mca_btl_ugni_module_setup_mpools (mca_btl_ugni_module_t *ugni_module)
{
struct mca_mpool_base_resources_t mpool_resources;
int mbox_increment, rc;
size_t nprocs;
(void) ompi_proc_world (&nprocs);
mpool_resources.reg_data = (void *) ugni_module;
mpool_resources.sizeof_reg = sizeof (mca_btl_ugni_reg_t);
mpool_resources.register_mem = ugni_reg_mem;
mpool_resources.deregister_mem = ugni_dereg_mem;
ugni_module->super.btl_mpool =
mca_mpool_base_module_create("rdma", ugni_module->device,
&mpool_resources);
if (NULL == ugni_module->super.btl_mpool) {
BTL_ERROR(("error creating mpool"));
return OMPI_ERROR;
}
mpool_resources.register_mem = ugni_reg_smsg_mem;
ugni_module->smsg_mpool =
mca_mpool_base_module_create("rdma", ugni_module->device,
&mpool_resources);
OBJ_CONSTRUCT(&ugni_module->smsg_mboxes, ompi_free_list_t);
mbox_increment = nprocs;
if (nprocs * mca_btl_ugni_smsg_mbox_size > 2 * 1024 * 1024) {
/* allocate at most 2 MB at a time (TODO: make this a mca param?) */
mbox_increment = (int) (2.0 * 1024.0 * 1024.0 / (float)mca_btl_ugni_smsg_mbox_size);
}
rc = ompi_free_list_init_new (&ugni_module->smsg_mboxes,
sizeof (mca_btl_ugni_smsg_mbox_t), 64,
OBJ_CLASS(mca_btl_ugni_smsg_mbox_t),
mca_btl_ugni_smsg_mbox_size,
opal_cache_line_size, 0,
nprocs, mbox_increment,
ugni_module->smsg_mpool);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
return rc;
}
return OMPI_SUCCESS;
}
int
mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
ompi_common_ugni_device_t *dev)
@ -214,13 +105,8 @@ mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
OBJ_CONSTRUCT(&ugni_module->failed_frags, opal_list_t);
/* module settings */
ugni_module->super.btl_eager_limit = mca_btl_ugni_component.eager_limit;
ugni_module->super.btl_max_send_size = ugni_module->super.btl_eager_limit;
ugni_module->super.btl_rdma_pipeline_send_length = ugni_module->super.btl_eager_limit;
ugni_module->device = dev;
ugni_module->endpoints = NULL;
/* create wildcard endpoint to listen for connections.
* there is no need to bind this endpoint. */
@ -238,8 +124,6 @@ mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
return rc;
}
ugni_module->endpoints = NULL;
rc = GNI_CqCreate (ugni_module->device->dev_handle, mca_btl_ugni_component.cq_size,
0, GNI_CQ_NOBLOCK, NULL, NULL, &ugni_module->bte_local_cq);
if (GNI_RC_SUCCESS != rc) {
@ -247,7 +131,6 @@ mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
return ompi_common_rc_ugni_to_ompi (rc);
}
/* the smsg_remote_cq must be created before we setup the smsg mpool */
rc = GNI_CqCreate (ugni_module->device->dev_handle, mca_btl_ugni_component.cq_size,
0, GNI_CQ_NOBLOCK, NULL, NULL, &ugni_module->smsg_remote_cq);
if (GNI_RC_SUCCESS != rc) {
@ -255,13 +138,6 @@ mca_btl_ugni_module_init (mca_btl_ugni_module_t *ugni_module,
return ompi_common_rc_ugni_to_ompi (rc);
}
/* create rdma and smsg mpools */
rc = mca_btl_ugni_module_setup_mpools (ugni_module);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
BTL_ERROR(("error setting up module mpools"));
return rc;
}
return OMPI_SUCCESS;
}
@ -272,6 +148,9 @@ mca_btl_ugni_module_finalize (struct mca_btl_base_module_t *btl)
size_t ntotal_procs, i;
int rc;
OBJ_DESTRUCT(&ugni_module->eager_frags_send);
OBJ_DESTRUCT(&ugni_module->eager_frags_recv);
/* close all open connections and release endpoints */
if (NULL != ugni_module->endpoints) {
(void) ompi_proc_world (&ntotal_procs);
@ -331,10 +210,11 @@ mca_btl_ugni_alloc(struct mca_btl_base_module_t *btl,
uint8_t order, size_t size, uint32_t flags)
{
mca_btl_ugni_base_frag_t *frag = NULL;
int rc = OMPI_SUCCESS;
if (size <= mca_btl_ugni_component.eager_limit) {
MCA_BTL_UGNI_FRAG_ALLOC_EAGER((mca_btl_ugni_module_t *) btl, frag, rc);
if (size <= mca_btl_ugni_component.smsg_max_data) {
(void) MCA_BTL_UGNI_FRAG_ALLOC_SMSG(endpoint, frag);
} else if (size <= btl->btl_eager_limit) {
(void) MCA_BTL_UGNI_FRAG_ALLOC_EAGER_SEND(endpoint, frag);
}
BTL_VERBOSE(("btl/ugni_module allocated frag of size: %u, flags: %x. frag = %p",
@ -347,6 +227,7 @@ mca_btl_ugni_alloc(struct mca_btl_base_module_t *btl,
frag->base.des_src_cnt = 1;
frag->base.des_dst = frag->segments;
frag->base.des_dst_cnt = 1;
frag->endpoint = endpoint;
frag->segments[0].seg_len = size;
}
@ -358,7 +239,7 @@ static int
mca_btl_ugni_free (struct mca_btl_base_module_t *btl,
mca_btl_base_descriptor_t *des)
{
MCA_BTL_UGNI_FRAG_RETURN((mca_btl_ugni_base_frag_t *) des);
mca_btl_ugni_frag_return ((mca_btl_ugni_base_frag_t *) des);
return OMPI_SUCCESS;
}
@ -379,12 +260,18 @@ mca_btl_ugni_prepare_src (struct mca_btl_base_module_t *btl,
opal_convertor_get_current_pointer (convertor, &data_ptr);
if (OPAL_LIKELY(reserve)) {
MCA_BTL_UGNI_FRAG_ALLOC_EAGER(ugni_module, frag, rc);
if ((*size + reserve) <= mca_btl_ugni_component.smsg_max_data) {
(void) MCA_BTL_UGNI_FRAG_ALLOC_SMSG(endpoint, frag);
} else {
(void) MCA_BTL_UGNI_FRAG_ALLOC_EAGER_SEND(endpoint, frag);
}
if (OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
if ((*size + reserve) > mca_btl_ugni_component.eager_limit) {
*size = mca_btl_ugni_component.eager_limit - reserve;
if ((*size + reserve) > btl->btl_eager_limit) {
*size = btl->btl_eager_limit - reserve;
}
BTL_VERBOSE(("preparing src for send fragment. size = %u",
@ -402,19 +289,18 @@ mca_btl_ugni_prepare_src (struct mca_btl_base_module_t *btl,
rc = opal_convertor_pack (convertor, &iov, &iov_count, size);
if (OPAL_UNLIKELY(rc < 0)) {
MCA_BTL_UGNI_FRAG_RETURN(frag);
mca_btl_ugni_frag_return (frag);
return NULL;
}
frag->segments[0].seg_len = reserve + *size;
}
else {
} else {
memmove ((void *)((uintptr_t)frag->segments[0].seg_addr.pval + reserve),
data_ptr, *size);
frag->segments[0].seg_len = reserve + *size;
}
} else {
MCA_BTL_UGNI_FRAG_ALLOC_RDMA(ugni_module, frag, rc);
(void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(endpoint, frag);
if (OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
@ -425,13 +311,12 @@ mca_btl_ugni_prepare_src (struct mca_btl_base_module_t *btl,
*/
/* No need to register while using FMA Put (registration is
* non-null in get-- is this always true?) */
if (*size >= mca_btl_ugni_component.btl_fma_limit || (flags & MCA_BTL_DES_FLAGS_GET)) {
if (*size >= mca_btl_ugni_component.ugni_fma_limit || (flags & MCA_BTL_DES_FLAGS_GET)) {
if (NULL == registration) {
rc = ugni_module->super.btl_mpool->mpool_register(ugni_module->super.btl_mpool,
data_ptr, *size, 0,
&registration);
rc = btl->btl_mpool->mpool_register(btl->btl_mpool, data_ptr,
*size, 0, &registration);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
MCA_BTL_UGNI_FRAG_RETURN(frag);
mca_btl_ugni_frag_return (frag);
return NULL;
}
@ -454,7 +339,6 @@ mca_btl_ugni_prepare_src (struct mca_btl_base_module_t *btl,
frag->base.des_src_cnt = 1;
frag->base.order = order;
frag->base.des_flags = flags;
frag->endpoint = endpoint;
return &frag->base;
}
@ -473,19 +357,18 @@ mca_btl_ugni_prepare_dst (mca_btl_base_module_t *btl,
opal_convertor_get_current_pointer (convertor, &data_ptr);
/* no alignment restrictions on put */
MCA_BTL_UGNI_FRAG_ALLOC_RDMA(ugni_module, frag, rc);
(void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(endpoint, frag);
if (OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
/* always need to register the buffer for put/get (even for fma) */
if (NULL == registration) {
rc = ugni_module->super.btl_mpool->mpool_register(ugni_module->super.btl_mpool,
data_ptr, *size, 0,
&registration);
rc = btl->btl_mpool->mpool_register(btl->btl_mpool,
data_ptr, *size, 0,
&registration);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
MCA_BTL_UGNI_FRAG_RETURN(frag);
mca_btl_ugni_frag_return (frag);
return NULL;
}
@ -503,7 +386,6 @@ mca_btl_ugni_prepare_dst (mca_btl_base_module_t *btl,
frag->base.des_dst_cnt = 1;
frag->base.order = order;
frag->base.des_flags = flags;
frag->endpoint = endpoint;
return (struct mca_btl_base_descriptor_t *) frag;
}

34
ompi/mca/btl/ugni/btl_ugni_put.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -26,40 +26,20 @@ int mca_btl_ugni_put (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
struct mca_btl_base_descriptor_t *des) {
mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) des;
gni_mem_handle_t lcl_hdl, rem_hdl;
void *lcl_buffer, *rem_buffer;
size_t size;
int rc;
BTL_VERBOSE(("Using RDMA Put"));
BTL_VERBOSE(("Using RDMA/FMA Put", ));
/* Check if endpoint is connected */
/* Check if endpoint is connected */
rc = mca_btl_ugni_check_endpoint_state(endpoint);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
/* we should already be connected by this point (we got an rc send) */
/* Ack! We should already be connected by this point (we got an smsg msg) */
return rc;
}
/* Get local memory handle */
lcl_buffer = (void*)(des->des_src->seg_addr.pval);
size = des->des_src->seg_len;
memcpy (&lcl_hdl, (void *) des->des_src->seg_key.key64,
sizeof(gni_mem_handle_t));
/* Get remote memory handle */
rem_buffer = (void*)(des->des_dst->seg_addr.pval);
memcpy (&rem_hdl, (void *) des->des_dst->seg_key.key64,
sizeof(gni_mem_handle_t));
frag->tries = 0;
if (size < mca_btl_ugni_component.btl_fma_limit) {
rc = post_fma_descriptor (frag, GNI_POST_FMA_PUT, endpoint, size,
lcl_buffer, lcl_hdl, rem_buffer, rem_hdl);
} else {
rc = post_bte_descriptor (frag, GNI_POST_RDMA_PUT, endpoint, size,
lcl_buffer, lcl_hdl, rem_buffer, rem_hdl);
if (frag->base.des_src->seg_len <= mca_btl_ugni_component.ugni_fma_limit) {
rc = mca_btl_ugni_post_fma (frag, GNI_POST_FMA_PUT, des->des_src, des->des_dst);
}
return rc;
return mca_btl_ugni_post_bte (frag, GNI_POST_RDMA_PUT, des->des_src, des->des_dst);
}

91
ompi/mca/btl/ugni/btl_ugni_rdma.h
Просмотреть файл

@ -15,22 +15,22 @@
#include "btl_ugni.h"
#include "btl_ugni_frag.h"
#include "btl_ugni_endpoint.h"
static inline void
mca_btl_ugni_post_frag_complete (ompi_common_ugni_post_desc_t *desc, int rc) {
mca_btl_ugni_base_frag_t *frag = MCA_BTL_UGNI_DESC_TO_FRAG(desc);
/* always call put/get callback */
frag->base.des_cbfunc(&frag->btl->super, frag->endpoint, &frag->base, rc);
/* always call put/get callback (if one is set) */
if (NULL != frag->base.des_cbfunc) {
frag->base.des_cbfunc(&frag->endpoint->btl->super, frag->endpoint, &frag->base, rc);
}
if (OPAL_LIKELY(frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
MCA_BTL_UGNI_FRAG_RETURN(frag);
mca_btl_ugni_frag_return (frag);
}
}
static inline int init_gni_post_desc(mca_btl_ugni_base_frag_t *frag,
mca_btl_base_endpoint_t *ep,
gni_post_type_t op_type,
uint64_t lcl_addr,
gni_mem_handle_t *lcl_mdh,
@ -38,59 +38,57 @@ static inline int init_gni_post_desc(mca_btl_ugni_base_frag_t *frag,
gni_mem_handle_t *rem_mdh,
uint64_t bufsize,
gni_cq_handle_t cq_hndl) {
frag->post_desc.base.type = op_type;
frag->post_desc.base.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
frag->post_desc.base.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
frag->post_desc.base.local_addr = (uint64_t) lcl_addr;
frag->post_desc.base.local_mem_hndl = *lcl_mdh;
frag->post_desc.base.remote_addr = (uint64_t) rem_addr;
frag->post_desc.base.type = op_type;
frag->post_desc.base.cq_mode = GNI_CQMODE_GLOBAL_EVENT;
frag->post_desc.base.dlvr_mode = GNI_DLVMODE_PERFORMANCE;
frag->post_desc.base.local_addr = (uint64_t) lcl_addr;
frag->post_desc.base.local_mem_hndl = *lcl_mdh;
frag->post_desc.base.remote_addr = (uint64_t) rem_addr;
frag->post_desc.base.remote_mem_hndl = *rem_mdh;
frag->post_desc.base.length = bufsize;
frag->post_desc.base.rdma_mode = 0;
frag->post_desc.base.src_cq_hndl = cq_hndl;
frag->post_desc.base.length = bufsize;
frag->post_desc.base.rdma_mode = 0;
frag->post_desc.base.src_cq_hndl = cq_hndl;
frag->post_desc.cbfunc = mca_btl_ugni_post_frag_complete;
frag->post_desc.endpoint = ep->common;
frag->post_desc.tries = 0;
frag->post_desc.cbfunc = mca_btl_ugni_post_frag_complete;
frag->post_desc.endpoint = frag->endpoint->common;
frag->post_desc.tries = 0;
return 0;
}
static inline int post_fma_descriptor (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
struct mca_btl_base_endpoint_t *endpoint,
size_t size, void *lcl_buffer, gni_mem_handle_t lcl_hdl,
void *rem_buffer, gni_mem_handle_t rem_hdl)
static inline int mca_btl_ugni_post_fma (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
mca_btl_base_segment_t *lcl_seg, mca_btl_base_segment_t *rem_seg)
{
int rc;
/* Post descriptor */
init_gni_post_desc (frag, endpoint, op_type, (uint64_t)lcl_buffer,
&lcl_hdl, (uint64_t)rem_buffer, &rem_hdl,
size, 0);
init_gni_post_desc (frag, op_type, lcl_seg->seg_addr.lval,
(gni_mem_handle_t *)&lcl_seg->seg_key.key64,
rem_seg->seg_addr.lval, (gni_mem_handle_t *)&rem_seg->seg_key.key64,
lcl_seg->seg_len, 0);
rc = GNI_PostFma (endpoint->common->ep_handle, &frag->post_desc.base);
rc = GNI_PostFma (frag->endpoint->common->ep_handle, &frag->post_desc.base);
if (GNI_RC_SUCCESS != rc) {
BTL_ERROR(("GNI_PostFma failed with rc = %d", rc));
assert(rc < 4);
rc = OMPI_ERR_OUT_OF_RESOURCE; /* ompi_common_rc_ugni_to_ompi (rc);*/
rc = OMPI_ERR_OUT_OF_RESOURCE;
}
return rc;
}
static inline int post_bte_descriptor (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
struct mca_btl_base_endpoint_t *endpoint,
size_t size, void *lcl_buffer, gni_mem_handle_t lcl_hdl,
void *rem_buffer, gni_mem_handle_t rem_hdl) {
static inline int mca_btl_ugni_post_bte (mca_btl_ugni_base_frag_t *frag, gni_post_type_t op_type,
mca_btl_base_segment_t *lcl_seg, mca_btl_base_segment_t *rem_seg)
{
int rc;
/* Post descriptor */
init_gni_post_desc (frag, endpoint, op_type, (uint64_t)lcl_buffer,
&lcl_hdl, (uint64_t)rem_buffer, &rem_hdl,
size, endpoint->btl->bte_local_cq);
init_gni_post_desc (frag, op_type, lcl_seg->seg_addr.lval,
(gni_mem_handle_t *)&lcl_seg->seg_key.key64,
rem_seg->seg_addr.lval, (gni_mem_handle_t *)&rem_seg->seg_key.key64,
lcl_seg->seg_len, frag->endpoint->btl->bte_local_cq);
rc = GNI_PostRdma (endpoint->common->ep_handle, &frag->post_desc.base);
rc = GNI_PostRdma (frag->endpoint->common->ep_handle, &frag->post_desc.base);
if (GNI_RC_SUCCESS != rc) {
assert(rc < 4);
rc = ompi_common_rc_ugni_to_ompi (rc);
@ -100,27 +98,4 @@ static inline int post_bte_descriptor (mca_btl_ugni_base_frag_t *frag, gni_post_
return rc;
}
static inline int mca_btl_ugni_start_reverse_get (struct mca_btl_base_module_t *btl,
mca_btl_ugni_base_frag_t *frag) {
/* off alignment/off size. switch to put */
mca_btl_base_segment_t segments[2];
void *post_desc_ptr = &(frag->post_desc);
int rc;
segments[0] = frag->base.des_src[0];
segments[1] = frag->base.des_dst[0];
rc = GNI_SmsgSendWTag (frag->endpoint->common->ep_handle, segments,
sizeof (segments), &post_desc_ptr, sizeof (void *),
-1, MCA_BTL_UGNI_TAG_PUT_INIT);
if (OPAL_UNLIKELY(rc == GNI_RC_NOT_DONE)) {
/* send this smsg packet later */
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* todo -- on failure try again */
assert (GNI_RC_SUCCESS == rc);
return OMPI_SUCCESS;
}
#endif /* MCA_BTL_UGNI_RDMA_H */

42
ompi/mca/btl/ugni/btl_ugni_send.c
Просмотреть файл

@ -12,7 +12,7 @@
#include "btl_ugni.h"
#include "btl_ugni_frag.h"
#include "btl_ugni_endpoint.h"
#include "btl_ugni_smsg.h"
void mca_btl_ugni_local_smsg_complete (void *btl_ctx, uint32_t msg_id, int rc)
{
@ -41,7 +41,7 @@ void mca_btl_ugni_local_smsg_complete (void *btl_ctx, uint32_t msg_id, int rc)
}
if (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP) {
MCA_BTL_UGNI_FRAG_RETURN (frag);
mca_btl_ugni_frag_return (frag);
}
}
@ -51,15 +51,19 @@ int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
mca_btl_base_tag_t tag)
{
mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) descriptor;
static uint8_t msg_num = 0;
int rc;
BTL_VERBOSE(("btl/ugni sending descriptor %p from %d -> %d. length = %d", (void *)descriptor,
ORTE_PROC_MY_NAME->vpid, btl_peer->common->ep_rem_id, frag->segments[0].seg_len));
/* tag and len are at the same location in eager and smsg frag hdrs */
frag->hdr.send.tag = tag;
frag->hdr.send.len = frag->segments[0].seg_len;
frag->endpoint = btl_peer;
rc = mca_btl_ugni_check_endpoint_state (btl_peer);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
frag->tag = tag;
descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
opal_list_append (&btl_peer->pending_list, (opal_list_item_t *) frag);
@ -67,27 +71,15 @@ int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
return OMPI_SUCCESS;
}
frag->hdr->tag = tag;
frag->hdr->len = frag->segments[0].seg_len;
frag->msg_id = (btl_peer->common->ep_rem_id & 0x00ffffff) | ((uint32_t)msg_num++ << 24) ;
if (frag->segments[0].seg_len <= mca_btl_ugni_component.smsg_max_data) {
return ompi_mca_btl_ugni_smsg_send (frag, false, &frag->hdr.send, sizeof (frag->hdr.send),
descriptor->des_src->seg_addr.pval, descriptor->des_src->seg_len,
MCA_BTL_UGNI_TAG_SEND);
} else {
frag->hdr.eager.src_seg = frag->segments[0];
frag->hdr.eager.ctx = (void *) &frag->post_desc;
/* check endpoint state */
rc = GNI_SmsgSendWTag (btl_peer->common->ep_handle, frag->hdr,
sizeof (frag->hdr[0]), descriptor->des_src->seg_addr.pval,
descriptor->des_src->seg_len, frag->msg_id, MCA_BTL_UGNI_TAG_SEND);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
BTL_VERBOSE(("GNI_SmsgSendWTag failed with rc = %d", rc));
if (OPAL_LIKELY(GNI_RC_NOT_DONE == rc)) {
BTL_VERBOSE(("out of credits"));
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_ERROR;
return ompi_mca_btl_ugni_smsg_send (frag, true, &frag->hdr.eager, sizeof (frag->hdr.eager),
NULL, 0, MCA_BTL_UGNI_TAG_GET_INIT);
}
opal_list_append (&btl_peer->pending_smsg_sends, (opal_list_item_t *) frag);
return 0;
}

72
ompi/mca/btl/ugni/btl_ugni_sendi.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -12,7 +12,7 @@
#include "btl_ugni.h"
#include "btl_ugni_frag.h"
#include "btl_ugni_endpoint.h"
#include "btl_ugni_smsg.h"
int mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
@ -24,14 +24,11 @@ int mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
{
size_t length = header_size + payload_size;
mca_btl_ugni_base_frag_t *frag;
uint32_t iov_count = 1;
void *data_ptr = NULL;
struct iovec iov;
size_t max_data;
int rc;
assert (length < mca_btl_ugni_component.eager_limit);
assert (0 == (flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK));
assert (length <= btl->btl_eager_limit && !(flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK));
if (OPAL_UNLIKELY(OMPI_SUCCESS != mca_btl_ugni_check_endpoint_state (endpoint))) {
/* can't complete inline send if the endpoint is not already connected */
@ -41,7 +38,12 @@ int mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
return OMPI_ERR_RESOURCE_BUSY;
}
MCA_BTL_UGNI_FRAG_ALLOC_EAGER((mca_btl_ugni_module_t *) btl, frag, rc);
if (length <= mca_btl_ugni_component.smsg_max_data) {
rc = MCA_BTL_UGNI_FRAG_ALLOC_SMSG(endpoint, frag);
} else {
rc = MCA_BTL_UGNI_FRAG_ALLOC_EAGER_SEND(endpoint, frag);
}
if (OPAL_UNLIKELY(NULL == frag)) {
*descriptor = NULL;
return OMPI_ERR_OUT_OF_RESOURCE;
@ -50,45 +52,47 @@ int mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
BTL_VERBOSE(("btl/ugni sending inline descriptor %p from %d -> %d. length = %u", (void *) frag,
ORTE_PROC_MY_NAME->vpid, endpoint->common->ep_rem_id, (unsigned int) length));
frag->base.des_cbfunc = NULL;
frag->base.des_flags = flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
frag->segments[0].seg_len = length;
frag->hdr.send.tag = tag;
frag->hdr.send.len = length;
/* write match header (with MPI comm/tag/etc. info) */
memcpy (frag->segments[0].seg_addr.pval, header, header_size);
frag->hdr->tag = tag;
frag->hdr->len = length;
/*
We can add MEMCHECKER calls before and after the packing.
*/
if (OPAL_UNLIKELY(payload_size && opal_convertor_need_buffers (convertor))) {
/* pack the data into the supplied buffer */
iov.iov_base = (IOVBASE_TYPE *)((uintptr_t)frag->segments[0].seg_addr.pval + header_size);
iov.iov_len = max_data = payload_size;
if (payload_size) {
if (OPAL_UNLIKELY(opal_convertor_need_buffers (convertor))) {
uint32_t iov_count = 1;
struct iovec iov;
(void) opal_convertor_pack (convertor, &iov, &iov_count, &max_data);
/* pack the data into the supplied buffer */
iov.iov_base = (IOVBASE_TYPE *)((uintptr_t)frag->segments[0].seg_addr.pval + header_size);
iov.iov_len = max_data = payload_size;
assert (max_data == payload_size);
(void) opal_convertor_pack (convertor, &iov, &iov_count, &max_data);
header_size += payload_size;
} else if (payload_size) {
opal_convertor_get_current_pointer (convertor, &data_ptr);
memmove ((uintptr_t)frag->segments[0].seg_addr.pval + header_size, data_ptr, payload_size);
assert (max_data == payload_size);
} else {
opal_convertor_get_current_pointer (convertor, &data_ptr);
memmove ((void *)((uintptr_t)frag->segments[0].seg_addr.pval + header_size), data_ptr, payload_size);
}
}
frag->base.des_cbfunc = NULL;
frag->msg_id = endpoint->common->ep_rem_id & 0x00ffffff;
/* send message */
rc = GNI_SmsgSendWTag (endpoint->common->ep_handle, frag->hdr, sizeof (frag->hdr[0]),
frag->segments[0].seg_addr.pval, length, frag->msg_id, MCA_BTL_UGNI_TAG_SEND);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
BTL_VERBOSE(("GNI_SmsgSendWTag failed with rc = %d", rc));
MCA_BTL_UGNI_FRAG_RETURN (frag);
*descriptor = NULL;
if (length <= mca_btl_ugni_component.smsg_max_data) {
return ompi_mca_btl_ugni_smsg_send (frag, false, &frag->hdr.send, sizeof (frag->hdr.send),
frag->segments[0].seg_addr.pval, length, MCA_BTL_UGNI_TAG_SEND);
} else {
frag->hdr.eager.src_seg = frag->segments[0];
frag->hdr.eager.ctx = (void *) &frag->post_desc;
return OMPI_ERR_OUT_OF_RESOURCE;
return ompi_mca_btl_ugni_smsg_send (frag, true, &frag->hdr.eager, sizeof (frag->hdr.eager),
NULL, 0, MCA_BTL_UGNI_TAG_GET_INIT);
}
opal_list_append (&endpoint->pending_smsg_sends, (opal_list_item_t *) frag);
return OMPI_SUCCESS;
}

67
ompi/mca/btl/ugni/btl_ugni_smsg.h Обычный файл
Просмотреть файл

@ -0,0 +1,67 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#if !defined(MCA_BTL_UGNI_SMSG_H)
#define MCA_BTL_UGNI_SMSG_H
#include "btl_ugni.h"
#include "btl_ugni_endpoint.h"
typedef enum {
MCA_BTL_UGNI_TAG_SEND,
MCA_BTL_UGNI_TAG_DISCONNECT,
MCA_BTL_UGNI_TAG_PUT_INIT,
MCA_BTL_UGNI_TAG_PUT_COMPLETE,
MCA_BTL_UGNI_TAG_GET_INIT,
MCA_BTL_UGNI_TAG_GET_COMPLETE
} mca_btl_ugni_smsg_tag_t;
typedef struct mca_btl_ugni_smsg_mbox_t {
ompi_free_list_item_t super;
gni_smsg_attr_t smsg_attrib;
} mca_btl_ugni_smsg_mbox_t;
OBJ_CLASS_DECLARATION(mca_btl_ugni_smsg_mbox_t);
static inline int ompi_mca_btl_ugni_smsg_send (mca_btl_ugni_base_frag_t *frag,
const bool ignore_local_comp,
void *hdr, size_t hdr_len,
void *payload, size_t payload_len,
mca_btl_ugni_smsg_tag_t tag) {
static uint8_t msg_num = 0;
gni_return_t rc;
frag->msg_id = ignore_local_comp ? (uint32_t) -1 :
(frag->endpoint->common->ep_rem_id & 0x00ffffff) | ((uint32_t)msg_num++ << 24);
rc = GNI_SmsgSendWTag (frag->endpoint->common->ep_handle, hdr, hdr_len, payload, payload_len,
frag->msg_id, tag);
if (OPAL_UNLIKELY(GNI_RC_SUCCESS != rc)) {
BTL_VERBOSE(("GNI_SmsgSendWTag failed with rc = %d", rc));
if (OPAL_LIKELY(GNI_RC_NOT_DONE == rc)) {
BTL_VERBOSE(("out of credits"));
return OMPI_ERR_OUT_OF_RESOURCE;
}
return OMPI_ERROR;
}
if (false == ignore_local_comp) {
opal_list_append (&frag->endpoint->pending_smsg_sends, (opal_list_item_t *) frag);
}
return OMPI_SUCCESS;
}
#endif /* MCA_BTL_UGNI_SMSG_H */