ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Adds support for Cross Memory Attach in the sm btl.

Просмотр исходного кода 
This feature can be enabled at compile time with --with-cma passed
to configure.

At runtime it is also necessary to add "--mca btl btl_sm_use_cma 1"
to the mpirun command.

If both CMA and KNEM are compiled in and enabled at runtime then 
KNEM will take precedence and CMA will disable itself

This commit was SVN r26134.
Этот коммит содержится в:
Christopher Yeoh 2012-03-14 06:29:09 +00:00
родитель a0e41ca685
Коммит 524de80eaa
4 изменённых файлов: 269 добавлений и 135 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

88
ompi/mca/btl/sm/btl_sm.c
Просмотреть файл

@ -13,6 +13,7 @@
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2010-2012 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -32,6 +33,10 @@
#include <sys/mman.h>
#endif /* HAVE_SYS_MMAN_H */
#ifdef OMPI_BTL_SM_CMA_NEED_SYSCALL_DEFS
#include "opal/sys/cma.h"
#endif /* OMPI_BTL_SM_CMA_NEED_SYSCALL_DEFS */
#include "opal/sys/atomic.h"
#include "opal/class/opal_bitmap.h"
#include "opal/util/output.h"
@ -80,11 +85,11 @@ mca_btl_sm_t mca_btl_sm = {
mca_btl_sm_alloc,
mca_btl_sm_free,
mca_btl_sm_prepare_src,
#if OMPI_BTL_SM_HAVE_KNEM
#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
mca_btl_sm_prepare_dst,
#else
NULL,
#endif /* OMPI_BTL_SM_HAVE_KNEM */
#endif /* OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA */
mca_btl_sm_send,
mca_btl_sm_sendi,
NULL, /* put */
@ -685,12 +690,12 @@ struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_src(
uint32_t iov_count = 1;
size_t max_data = *size;
int rc;
#if OMPI_BTL_SM_HAVE_KNEM
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*)btl;
struct knem_cmd_create_region knem_cr;
struct knem_cmd_param_iovec knem_iov;
if( (0 != reserve) || (OPAL_UNLIKELY(!mca_btl_sm_component.use_knem)) ) {
#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*)btl;
if( (0 != reserve) || ( OPAL_UNLIKELY(!mca_btl_sm_component.use_knem)
&& OPAL_UNLIKELY(!mca_btl_sm_component.use_cma)) ) {
#endif
if ( reserve + max_data <= mca_btl_sm_component.eager_limit ) {
MCA_BTL_SM_FRAG_ALLOC_EAGER(frag,rc);
@ -715,8 +720,12 @@ struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_src(
return NULL;
}
frag->segment.seg_len = reserve + max_data;
#if OMPI_BTL_SM_HAVE_KNEM
#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
} else {
#if OMPI_BTL_SM_HAVE_KNEM
struct knem_cmd_create_region knem_cr;
struct knem_cmd_param_iovec knem_iov;
#endif /* OMPI_BTL_SM_HAVE_KNEM */
MCA_BTL_SM_FRAG_ALLOC_USER(frag, rc);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
@ -731,6 +740,8 @@ struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_src(
frag->segment.seg_addr.pval = iov.iov_base;
frag->segment.seg_len = max_data;
#if OMPI_BTL_SM_HAVE_KNEM
if (OPAL_LIKELY(mca_btl_sm_component.use_knem)) {
knem_iov.base = (uintptr_t)iov.iov_base;
knem_iov.len = max_data;
knem_cr.iovec_array = (uintptr_t)&knem_iov;
@ -742,7 +753,17 @@ struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_src(
}
frag->segment.seg_key.key64[0] = knem_cr.cookie;
}
#endif
#endif /* OMPI_BTL_SM_HAVE_KNEM */
#if OMPI_BTL_SM_HAVE_CMA
if (OPAL_LIKELY(mca_btl_sm_component.use_cma)) {
/* Encode the pid as the key */
frag->segment.seg_key.key64[0] = getpid();
}
#endif /* OMPI_BTL_SM_HAVE_CMA */
}
#endif /* OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA */
frag->base.des_src = &(frag->segment);
frag->base.des_src_cnt = 1;
frag->base.order = MCA_BTL_NO_ORDER;
@ -914,7 +935,7 @@ int mca_btl_sm_send( struct mca_btl_base_module_t* btl,
return 0;
}
#if OMPI_BTL_SM_HAVE_KNEM
#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_dst(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
@ -960,6 +981,8 @@ int mca_btl_sm_get_sync(struct mca_btl_base_module_t* btl,
mca_btl_sm_frag_t* frag = (mca_btl_sm_frag_t*)des;
mca_btl_base_segment_t *src = des->des_src;
mca_btl_base_segment_t *dst = des->des_dst;
#if OMPI_BTL_SM_HAVE_KNEM
if (OPAL_LIKELY(mca_btl_sm_component.use_knem)) {
struct knem_cmd_inline_copy icopy;
struct knem_cmd_param_iovec recv_iovec;
@ -991,6 +1014,44 @@ int mca_btl_sm_get_sync(struct mca_btl_base_module_t* btl,
}
/* FIXME: what if icopy.current_status == KNEM_STATUS_FAILED? */
}
#endif
#if OMPI_BTL_SM_HAVE_CMA
if (OPAL_LIKELY(mca_btl_sm_component.use_cma)) {
char *remote_address, *local_address;
int remote_length, local_length;
struct iovec local, remote;
pid_t remote_pid;
int val;
remote_address = (char *) src->seg_addr.pval;
remote_length = src->seg_len;
local_address = (char *) dst->seg_addr.pval;
local_length = dst->seg_len;
remote_pid = src->seg_key.key64[0];
remote.iov_base = src->seg_addr.pval;
remote.iov_len = src->seg_len;
local.iov_base = dst->seg_addr.pval;
local.iov_len = dst->seg_len;
val = process_vm_readv(remote_pid, &local, 1, &remote, 1, 0);
if (val != local_length) {
if (val<0) {
opal_output(0, "mca_btl_sm_get_sync: process_vm_readv failed: %i",
errno);
} else {
/* Should never get a short read from process_vm_readv */
opal_output(0, "mca_btl_sm_get_sync: process_vm_readv short read: %i",
val);
}
return OMPI_ERROR;
}
}
#endif /* OMPI_BTL_SM_HAVE_CMA */
btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if (0 != (MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags)) {
@ -1005,6 +1066,11 @@ int mca_btl_sm_get_sync(struct mca_btl_base_module_t* btl,
return OMPI_SUCCESS;
}
#endif /* OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA */
/* No support async_get for CMA yet */
#if OMPI_BTL_SM_HAVE_KNEM
/**
* Initiate an asynchronous get.
@ -1085,7 +1151,7 @@ int mca_btl_sm_get_async(struct mca_btl_base_module_t* btl,
return OMPI_ERROR;
}
}
#endif
#endif /* OMPI_BTL_SM_HAVE_KNEM */
#if OPAL_ENABLE_FT_CR == 0

27
ompi/mca/btl/sm/btl_sm.h
Просмотреть файл

@ -13,6 +13,7 @@
* Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2010 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2010-2012 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -197,6 +198,10 @@ struct mca_btl_sm_component_t {
/** If we want DMA and DMA is supported, this will be loaded with
KNEM_FLAG_DMA. Otherwise, it'll be 0. */
int knem_dma_flag;
/** MCA: should we be using CMA or not?
0 = no, 1 = yes */
int use_cma;
};
typedef struct mca_btl_sm_component_t mca_btl_sm_component_t;
OMPI_MODULE_DECLSPEC extern mca_btl_sm_component_t mca_btl_sm_component;
@ -492,21 +497,14 @@ extern int mca_btl_sm_send(
mca_btl_base_tag_t tag
);
#if OMPI_BTL_SM_HAVE_KNEM
#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
/*
* Synchronous knem get
* Synchronous knem/cma get
*/
extern int mca_btl_sm_get_sync(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des );
/*
* Asynchronous knem get
*/
extern int mca_btl_sm_get_async(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des );
extern struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_dst(
struct mca_btl_base_module_t* btl,
@ -519,6 +517,17 @@ extern struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_dst(
uint32_t flags);
#endif
#if OMPI_BTL_SM_HAVE_KNEM
/*
* Asynchronous knem get
*/
extern int mca_btl_sm_get_async(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des );
#endif
/**
* Fault Tolerance Event Notification Function
* @param state Checkpoint Stae

31
ompi/mca/btl/sm/btl_sm_component.c
Просмотреть файл

@ -14,6 +14,7 @@
* Copyright (c) 2010-2011 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2010-2012 IBM Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -172,6 +173,13 @@ static int sm_register(void)
false, false, 0,
&mca_btl_sm_component.knem_max_simultaneous);
/* CMA parameters */
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
"use_cma",
"Whether or not to enable CMA",
false, false, 0, &mca_btl_sm_component.use_cma);
/* register SM component parameters */
mca_btl_sm_component.sm_free_list_num =
mca_btl_sm_param_register_int("free_list_num", 8);
@ -203,10 +211,17 @@ static int sm_register(void)
mca_btl_sm.super.btl_rdma_pipeline_frag_size = 64*1024;
mca_btl_sm.super.btl_min_rdma_pipeline_size = 64*1024;
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND;
#if OMPI_BTL_SM_HAVE_KNEM
if (mca_btl_sm_component.use_knem) {
#if OMPI_BTL_SM_HAVE_KNEM || OMPI_BTL_SM_HAVE_CMA
if (mca_btl_sm_component.use_knem || mca_btl_sm_component.use_cma) {
mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_GET;
}
if (mca_btl_sm_component.use_knem && mca_btl_sm_component.use_cma) {
/* Disable CMA if knem is runtime enabled */
opal_output(0, "CMA disabled because knem is enabled");
mca_btl_sm_component.use_cma = 0;
}
#endif
mca_btl_sm.super.btl_bandwidth = 9000; /* Mbs */
mca_btl_sm.super.btl_latency = 1; /* Microsecs */
@ -405,6 +420,7 @@ static mca_btl_base_module_t** mca_btl_sm_component_init(
mca_btl_sm.btl_inited = false;
#if OMPI_BTL_SM_HAVE_KNEM
if (mca_btl_sm_component.use_knem) {
/* Set knem_status_num_used outside the check for use_knem so that
we can only have to check one thing (knem_status_num_used) in
the progress loop. */
@ -494,7 +510,16 @@ static mca_btl_base_module_t** mca_btl_sm_component_init(
} else {
mca_btl_sm.super.btl_get = mca_btl_sm_get_sync;
}
#endif
}
#endif /* OMPI_BTL_SM_HAVE_KNEM */
#if OMPI_BTL_SM_HAVE_CMA
if (mca_btl_sm_component.use_cma) {
/* Will only ever have either cma or knem enabled at runtime
so no problems with accidentally overwriting this set earlier */
mca_btl_sm.super.btl_get = mca_btl_sm_get_sync;
}
#endif /* OMPI_BTL_SM_HAVE_CMA */
return btls;

34
ompi/mca/btl/sm/configure.m4
Просмотреть файл

@ -4,6 +4,7 @@
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved.
# Copyright (c) 2010-2012 IBM Corporation. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -11,6 +12,30 @@
# $HEADER$
#
# OMPI_CHECK_CMA(prefix, [action-if-found], [action-if-not-found])
# --------------------------------------------------------
# check if cma support is wanted.
AC_DEFUN([OMPI_CHECK_CMA],[
AC_ARG_WITH([cma],
[AC_HELP_STRING([--with-cma],
[Build Cross Memory Attach support (default: no)])])
AC_MSG_CHECKING([if user requested CMA build])
if test "$with_cma" = "yes" ; then
btl_sm_cma_happy="yes"
AC_CHECK_FUNC(process_vm_readv, [btl_sm_cma_need_defs=0],
[btl_sm_cma_need_defs=1])
AC_DEFINE_UNQUOTED([OMPI_BTL_SM_CMA_NEED_SYSCALL_DEFS],
[$btl_sm_cma_need_defs],
[Need CMA syscalls defined])
fi
AS_IF([test "$btl_sm_cma_happy" = "yes"],
[$2],
[$3])
])dnl
# OMPI_CHECK_KNEM(prefix, [action-if-found], [action-if-not-found])
# --------------------------------------------------------
# check if knem support can be found. sets prefix_{CPPFLAGS,
@ -69,6 +94,15 @@ AC_DEFUN([OMPI_CHECK_KNEM],[
AC_DEFUN([MCA_ompi_btl_sm_CONFIG],[
AC_CONFIG_FILES([ompi/mca/btl/sm/Makefile])
OPAL_VAR_SCOPE_PUSH([btl_sm_cma_happy])
OMPI_CHECK_CMA([btl_sm], [btl_sm_cma_happy=1], [btl_sm_cma_happy=0])
AC_DEFINE_UNQUOTED([OMPI_BTL_SM_HAVE_CMA],
[$btl_sm_cma_happy],
[If CMA support can be enabled])
OPAL_VAR_SCOPE_POP
OPAL_VAR_SCOPE_PUSH([btl_sm_knem_happy])
OMPI_CHECK_KNEM([btl_sm],
[btl_sm_knem_happy=1],