ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

btl/ugni: re-add a sendi function to exploit the new optimization in

Просмотр исходного кода 
ob1.

Also update LANL platform files to use the latest version of ugni.

cmr=v1.7.5:reviewer=manjugv

This commit was SVN r30353.
Этот коммит содержится в:
Nathan Hjelm 2014-01-21 20:53:35 +00:00
родитель 34b3718042
Коммит aa3fea55b2
5 изменённых файлов: 113 добавлений и 18 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

8
contrib/platform/lanl/cray_xe6/cray-common
Просмотреть файл

@ -1,4 +1,4 @@
# (c) 2012-2013 Los Alamos National Security, LLC. All rights reserved.
# (c) 2012-2014 Los Alamos National Security, LLC. All rights reserved.
# Common Cray XE/XK-6 options (used by all builds)
# do not use IB verbs
@ -7,11 +7,13 @@ with_verbs=no
# enable XPMEM enhanced shared memory (needs for Vader BTL)
with_xpmem=/opt/cray/xpmem/0.1-2.0400.30792.5.6.gem
# enable Cray PMI support
# enable Cray PMI support. Use an older version because of a regression
# in PMI 3.x/4.x/5.x that causes a bunch of irrelevant messages to be
# printed out on fork/exec.
with_pmi=/opt/cray/pmi/2.1.4-1.0000.8596.8.9.gem
# enable ugni btl
with_ugni=/opt/cray/ugni/2.3-1.0400.4127.5.20.gem
with_ugni=/opt/cray/ugni/4.0-1.0401.5928.9.5.gem
with_ugni_includedir=/opt/cray/gni-headers/default/include
# enable the udreg mpool

6
ompi/mca/btl/ugni/btl_ugni_frag.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -24,8 +24,8 @@ static inline void mca_btl_ugni_eager_frag_constructor (mca_btl_ugni_base_frag_t
struct mca_btl_ugni_reg_t *reg =
(struct mca_btl_ugni_reg_t *) frag->base.super.registration;
memset ((char *) frag + sizeof (frag->base), 0, sizeof (*frag) - sizeof (frag->base));
frag->segments[0].base.seg_addr.pval = frag->base.super.ptr;
mca_btl_ugni_base_frag_constructor (frag);
frag->segments[0].memory_handle = reg->memory_hdl;
}

3
ompi/mca/btl/ugni/btl_ugni_module.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -51,6 +51,7 @@ mca_btl_ugni_module_t mca_btl_ugni_module = {
.btl_prepare_src = mca_btl_ugni_prepare_src,
.btl_prepare_dst = mca_btl_ugni_prepare_dst,
.btl_send = mca_btl_ugni_send,
.btl_sendi = mca_btl_ugni_sendi,
.btl_put = mca_btl_ugni_put,
.btl_get = mca_btl_ugni_get,
}

53
ompi/mca/btl/ugni/btl_ugni_prepare.h
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -18,6 +18,39 @@
#include "btl_ugni.h"
#include "btl_ugni_frag.h"
static inline struct mca_btl_base_descriptor_t *
mca_btl_ugni_prepare_src_send_nodata (struct mca_btl_base_module_t *btl,
mca_btl_base_endpoint_t *endpoint,
uint8_t order, size_t reserve,
uint32_t flags)
{
mca_btl_ugni_base_frag_t *frag = NULL;
int rc;
(void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(endpoint, frag);
if (OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
BTL_VERBOSE(("preparing src for send fragment. size = %u",
(unsigned int)(*size + reserve)));
frag->hdr_size = reserve + sizeof (frag->hdr.send);
frag->segments[0].base.seg_addr.pval = frag->hdr.send_ex.pml_header;
frag->segments[0].base.seg_len = reserve;
frag->segments[1].base.seg_addr.pval = NULL;
frag->segments[1].base.seg_len = 0;
frag->base.des_src = &frag->segments->base;
frag->base.des_src_cnt = 1;
frag->base.order = order;
frag->base.des_flags = flags;
return &frag->base;
}
static inline struct mca_btl_base_descriptor_t *
mca_btl_ugni_prepare_src_send_inplace (struct mca_btl_base_module_t *btl,
mca_btl_base_endpoint_t *endpoint,
@ -115,15 +148,13 @@ mca_btl_ugni_prepare_src_send_buffered (struct mca_btl_base_module_t *btl,
frag->flags |= MCA_BTL_UGNI_FRAG_BUFFERED;
if (*size) {
iov.iov_len = *size;
iov.iov_base = (IOVBASE_TYPE *) frag->base.super.ptr;
iov.iov_len = *size;
iov.iov_base = (IOVBASE_TYPE *) frag->base.super.ptr;
rc = opal_convertor_pack (convertor, &iov, &iov_count, size);
if (OPAL_UNLIKELY(rc < 0)) {
mca_btl_ugni_frag_return (frag);
return NULL;
}
rc = opal_convertor_pack (convertor, &iov, &iov_count, size);
if (OPAL_UNLIKELY(rc < 0)) {
mca_btl_ugni_frag_return (frag);
return NULL;
}
frag->segments[0].base.seg_len = reserve;
@ -150,6 +181,10 @@ mca_btl_ugni_prepare_src_send (struct mca_btl_base_module_t *btl,
bool send_in_place;
void *data_ptr;
if (!(*size)) {
return mca_btl_ugni_prepare_src_send_nodata (btl, endpoint, order, reserve, flags);
}
opal_convertor_get_current_pointer (convertor, &data_ptr);
send_in_place = !(opal_convertor_need_buffers(convertor) ||

61
ompi/mca/btl/ugni/btl_ugni_send.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights
* Copyright (c) 2011-2014 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
* $COPYRIGHT$
@ -13,6 +13,7 @@
#include "btl_ugni.h"
#include "btl_ugni_frag.h"
#include "btl_ugni_smsg.h"
#include "btl_ugni_prepare.h"
int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
@ -27,7 +28,6 @@ int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
/* tag and len are at the same location in eager and smsg frag hdrs */
frag->hdr.send.lag = (tag << 24) | size;
frag->endpoint = endpoint;
rc = mca_btl_ugni_check_endpoint_state (endpoint);
if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
@ -82,6 +82,63 @@ int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
return rc;
}
int
mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
struct mca_btl_base_endpoint_t *endpoint,
struct opal_convertor_t *convertor,
void *header, size_t header_size,
size_t payload_size, uint8_t order,
uint32_t flags, mca_btl_base_tag_t tag,
mca_btl_base_descriptor_t **descriptor)
{
size_t total_size = header_size + payload_size;
mca_btl_ugni_base_frag_t *frag = NULL;
size_t packed_size = payload_size;
int rc;
do {
if (OPAL_UNLIKELY(OMPI_SUCCESS != mca_btl_ugni_check_endpoint_state (endpoint))) {
break;
}
frag = mca_btl_ugni_prepare_src_send (btl, endpoint, convertor, order, header_size,
&packed_size, flags & ~MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if (OPAL_UNLIKELY(NULL == frag || packed_size != payload_size)) {
break;
}
frag->hdr.send.lag = (tag << 24) | total_size;
memcpy (frag->segments[0].base.seg_addr.pval, header, header_size);
rc = mca_btl_ugni_send_frag (endpoint, frag);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
break;
}
if (!(frag->flags & MCA_BTL_UGNI_FRAG_BUFFERED)) {
/* wait until the frag is received by the remote end */
while (!(frag->flags & MCA_BTL_UGNI_FRAG_COMPLETE)) {
mca_btl_ugni_progress_local_smsg ((mca_btl_ugni_module_t *) btl);
}
}
frag->flags |= MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
if (frag->flags & MCA_BTL_UGNI_FRAG_COMPLETE) {
mca_btl_ugni_frag_return (frag);
}
return OMPI_SUCCESS;
} while (0);
if (frag) {
mca_btl_ugni_frag_return (frag);
}
*descriptor = NULL;
return OMPI_ERR_OUT_OF_RESOURCE;
}
int mca_btl_progress_send_wait_list (mca_btl_base_endpoint_t *endpoint)
{
mca_btl_ugni_base_frag_t *frag;