ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Enable support for the INRIA

Просмотр исходного кода 
knem (http://runtime.bordeaux.inria.fr/knem/) kernel device. This
is part of Ma Teng's work on Open MPI.

This commit was SVN r22315.
Этот коммит содержится в:
George Bosilca 2009-12-15 23:34:09 +00:00
родитель 9fa5f1d7a8
Коммит b85c3ca081
8 изменённых файлов: 913 добавлений и 57 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
ompi/mca/btl/sm/Makefile.am
Просмотреть файл

@ -2,13 +2,14 @@
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
# University Research and Technology
# Corporation. All rights reserved.
# Copyright (c) 2004-2005 The University of Tennessee and The University
# Copyright (c) 2004-2009 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
@ -18,6 +19,8 @@
EXTRA_DIST = .windows
dist_pkgdata_DATA = help-mpi-btl-sm.txt
libmca_btl_sm_la_sources = \
btl_sm.c \
btl_sm.h \
@ -48,7 +51,9 @@ mca_btl_sm_la_SOURCES = $(libmca_btl_sm_la_sources)
mca_btl_sm_la_LDFLAGS = -module -avoid-version
mca_btl_sm_la_LIBADD = \
$(top_ompi_builddir)/ompi/mca/common/sm/libmca_common_sm.la
mca_btl_sm_la_CPPFLAGS = $(btl_sm_CPPFLAGS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_btl_sm_la_SOURCES = $(libmca_btl_sm_la_sources)
libmca_btl_sm_la_LDFLAGS = -module -avoid-version
libmca_btl_sm_la_CPPFLAGS = $(btl_sm_CPPFLAGS)

277
ompi/mca/btl/sm/btl_sm.c
Просмотреть файл

@ -2,7 +2,7 @@
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@ -10,6 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Voltaire. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -42,6 +43,10 @@
#include "ompi/mca/common/sm/common_sm_mmap.h"
#include "ompi/mca/mpool/sm/mpool_sm.h"
#if OMPI_BTL_SM_HAVE_KNEM
#include <knem_io.h>
#endif
#if OPAL_ENABLE_FT == 1
#include "opal/mca/crs/base/base.h"
#include "opal/util/basename.h"
@ -74,11 +79,15 @@ mca_btl_sm_t mca_btl_sm = {
mca_btl_sm_alloc,
mca_btl_sm_free,
mca_btl_sm_prepare_src,
#if OMPI_BTL_SM_HAVE_KNEM
mca_btl_sm_prepare_dst,
#else
NULL,
#endif /* OMPI_BTL_SM_HAVE_KNEM */
mca_btl_sm_send,
mca_btl_sm_sendi,
NULL, /* put */
NULL, /* get */
NULL, /* get -- optionally filled during initialization */
mca_btl_base_dump,
NULL, /* mpool */
mca_btl_sm_register_error_cb, /* register error */
@ -349,6 +358,17 @@ static int sm_btl_first_time_init(mca_btl_sm_t *sm_btl, int n)
if ( OMPI_SUCCESS != i )
return i;
i = ompi_free_list_init_new(&mca_btl_sm_component.sm_frags_user,
sizeof(mca_btl_sm_user_t),
CACHE_LINE_SIZE, OBJ_CLASS(mca_btl_sm_user_t),
sizeof(mca_btl_sm_hdr_t), CACHE_LINE_SIZE,
mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc,
mca_btl_sm_component.sm_mpool);
if ( OMPI_SUCCESS != i )
return i;
mca_btl_sm_component.num_outstanding_frags = 0;
mca_btl_sm_component.num_pending_sends = 0;
@ -668,30 +688,69 @@ struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_src(
uint32_t iov_count = 1;
size_t max_data = *size;
int rc;
#if OMPI_BTL_SM_HAVE_KNEM
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*)btl;
struct knem_cmd_create_region knem_cr;
struct knem_cmd_param_iovec knem_iov;
if ( reserve + max_data <= mca_btl_sm_component.eager_limit ) {
MCA_BTL_SM_FRAG_ALLOC_EAGER(frag,rc);
if( (0 != reserve) || (OPAL_UNLIKELY(!mca_btl_sm_component.use_knem)) ) {
#endif
if ( reserve + max_data <= mca_btl_sm_component.eager_limit ) {
MCA_BTL_SM_FRAG_ALLOC_EAGER(frag,rc);
} else {
MCA_BTL_SM_FRAG_ALLOC_MAX(frag, rc);
}
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
if( OPAL_UNLIKELY(reserve + max_data > frag->size) ) {
max_data = frag->size - reserve;
}
iov.iov_len = max_data;
iov.iov_base =
(IOVBASE_TYPE*)(((unsigned char*)(frag->segment.seg_addr.pval)) +
reserve);
rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
if( OPAL_UNLIKELY(rc < 0) ) {
MCA_BTL_SM_FRAG_RETURN(frag);
return NULL;
}
frag->segment.seg_len = reserve + max_data;
#if OMPI_BTL_SM_HAVE_KNEM
} else {
MCA_BTL_SM_FRAG_ALLOC_MAX(frag, rc);
}
if(OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
MCA_BTL_SM_FRAG_ALLOC_USER(frag, rc);
if( OPAL_UNLIKELY(NULL == frag) ) {
return NULL;
}
iov.iov_len = max_data;
iov.iov_base = NULL;
rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data);
if( OPAL_UNLIKELY(rc < 0) ) {
MCA_BTL_SM_FRAG_RETURN(frag);
return NULL;
}
frag->segment.seg_addr.pval = iov.iov_base;
frag->segment.seg_len = max_data;
if(reserve + max_data > frag->size) {
max_data = frag->size - reserve;
knem_iov.base = (uintptr_t)iov.iov_base;
knem_iov.len = max_data;
knem_cr.iovec_array = (uintptr_t)&knem_iov;
knem_cr.iovec_nr = iov_count;
knem_cr.protection = PROT_READ;
knem_cr.flags = KNEM_FLAG_SINGLEUSE;
if (OPAL_UNLIKELY(ioctl(sm_btl->knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
return NULL;
}
frag->segment.seg_key.key64 = knem_cr.cookie;
}
iov.iov_len = max_data;
iov.iov_base =
(IOVBASE_TYPE*)(((unsigned char*)(frag->segment.seg_addr.pval)) +
reserve);
rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
if(rc < 0) {
MCA_BTL_SM_FRAG_RETURN(frag);
return NULL;
}
frag->segment.seg_len = reserve + max_data;
#endif
frag->base.des_src = &(frag->segment);
frag->base.des_src_cnt = 1;
frag->base.order = MCA_BTL_NO_ORDER;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = flags;
*size = max_data;
return &frag->base;
@ -858,6 +917,180 @@ int mca_btl_sm_send( struct mca_btl_base_module_t* btl,
return 0;
}
#if OMPI_BTL_SM_HAVE_KNEM
struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_dst(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags)
{
int rc;
mca_btl_sm_frag_t* frag;
MCA_BTL_SM_FRAG_ALLOC_USER(frag, rc);
if(OPAL_UNLIKELY(NULL == frag)) {
return NULL;
}
frag->segment.seg_len = *size;
opal_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->base.des_dst = &frag->segment;
frag->base.des_dst_cnt = 1;
frag->base.des_flags = flags;
return &frag->base;
}
/**
* Initiate an synchronous get.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
int mca_btl_sm_get_sync(struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des)
{
int btl_ownership;
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*) btl;
mca_btl_sm_frag_t* frag = (mca_btl_sm_frag_t*)des;
mca_btl_base_segment_t *src = des->des_src;
mca_btl_base_segment_t *dst = des->des_dst;
struct knem_cmd_inline_copy icopy;
struct knem_cmd_param_iovec recv_iovec;
/* Fill in the ioctl data fields. There's no async completion, so
we don't need to worry about getting a slot, etc. */
recv_iovec.base = (uintptr_t) dst->seg_addr.pval;
recv_iovec.len = dst->seg_len;
icopy.local_iovec_array = (uintptr_t)&recv_iovec;
icopy.local_iovec_nr = 1;
icopy.remote_cookie = src->seg_key.key64;
icopy.remote_offset = 0;
icopy.write = 0;
/* Use the DMA flag if knem supports it *and* the segment length
is greater than the cutoff. Note that if the knem_dma_min
value is 0 (i.e., the MCA param was set to 0), the segment size
will never be larger than it, so DMA will never be used. */
icopy.flags = 0;
if (mca_btl_sm_component.knem_dma_min <= dst->seg_len) {
icopy.flags = mca_btl_sm_component.knem_dma_flag;
}
/* synchronous flags only, no need to specify icopy.async_status_index */
/* When the ioctl returns, the transfer is done and we can invoke
the btl callback and return the frag */
if (OPAL_UNLIKELY(0 != ioctl(sm_btl->knem_fd,
KNEM_CMD_INLINE_COPY, &icopy))) {
return OMPI_ERROR;
}
/* FIXME: what if icopy.current_status == KNEM_STATUS_FAILED? */
btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if (0 != (MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags)) {
frag->base.des_cbfunc(&mca_btl_sm.super,
frag->endpoint, &frag->base,
OMPI_SUCCESS);
}
if (btl_ownership) {
MCA_BTL_SM_FRAG_RETURN(frag);
}
return OMPI_SUCCESS;
}
/**
* Initiate an asynchronous get.
*
* @param btl (IN) BTL module
* @param endpoint (IN) BTL addressing information
* @param descriptor (IN) Description of the data to be transferred
*/
int mca_btl_sm_get_async(struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des)
{
int btl_ownership;
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*) btl;
mca_btl_sm_frag_t* frag = (mca_btl_sm_frag_t*)des;
mca_btl_base_segment_t *src = des->des_src;
mca_btl_base_segment_t *dst = des->des_dst;
struct knem_cmd_inline_copy icopy;
struct knem_cmd_param_iovec recv_iovec;
/* If we have no knem slots available, return
TEMP_OUT_OF_RESOURCE */
if (sm_btl->knem_status_num_used >=
mca_btl_sm_component.knem_max_simultaneous) {
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
}
/* We have a slot, so fill in the data fields. Bump the
first_avail and num_used counters. */
recv_iovec.base = (uintptr_t) dst->seg_addr.pval;
recv_iovec.len = dst->seg_len;
icopy.local_iovec_array = (uintptr_t)&recv_iovec;
icopy.local_iovec_nr = 1;
icopy.write = 0;
icopy.async_status_index = sm_btl->knem_status_first_avail++;
if (sm_btl->knem_status_first_avail >=
mca_btl_sm_component.knem_max_simultaneous) {
sm_btl->knem_status_first_avail = 0;
}
++sm_btl->knem_status_num_used;
icopy.remote_cookie = src->seg_key.key64;
icopy.remote_offset = 0;
/* Use the DMA flag if knem supports it *and* the segment length
is greater than the cutoff */
icopy.flags = KNEM_FLAG_ASYNCDMACOMPLETE;
if (mca_btl_sm_component.knem_dma_min <= dst->seg_len) {
icopy.flags = mca_btl_sm_component.knem_dma_flag;
}
sm_btl->knem_frag_array[icopy.async_status_index] = frag;
if (OPAL_LIKELY(0 == ioctl(sm_btl->knem_fd,
KNEM_CMD_INLINE_COPY, &icopy))) {
if (icopy.current_status != KNEM_STATUS_PENDING) {
/* request completed synchronously */
/* FIXME: what if icopy.current_status == KNEM_STATUS_FAILED? */
btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if (0 != (MCA_BTL_DES_SEND_ALWAYS_CALLBACK & frag->base.des_flags)) {
frag->base.des_cbfunc(&mca_btl_sm.super,
frag->endpoint, &frag->base,
OMPI_SUCCESS);
}
if (btl_ownership) {
MCA_BTL_SM_FRAG_RETURN(frag);
}
--sm_btl->knem_status_num_used;
++sm_btl->knem_status_first_used;
if (sm_btl->knem_status_first_used >=
mca_btl_sm_component.knem_max_simultaneous) {
sm_btl->knem_status_first_used = 0;
}
}
return OMPI_SUCCESS;
} else {
return OMPI_ERROR;
}
}
#endif
#if OPAL_ENABLE_FT == 0
int mca_btl_sm_ft_event(int state) {
return OMPI_SUCCESS;

114
ompi/mca/btl/sm/btl_sm.h
Просмотреть файл

@ -3,7 +3,7 @@
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2006 The University of Tennessee and The University
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@ -11,6 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2006-2007 Voltaire. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -26,6 +27,17 @@
#include "ompi_config.h"
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <sched.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <fcntl.h>
#include <unistd.h>
#if OMPI_BTL_SM_HAVE_KNEM
#include "knem_io.h"
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif /* HAVE_SYS_TYPES_H */
@ -150,6 +162,7 @@ struct mca_btl_sm_component_t {
* SMP specfic data structures. */
ompi_free_list_t sm_frags_eager; /**< free list of sm first */
ompi_free_list_t sm_frags_max; /**< free list of sm second */
ompi_free_list_t sm_frags_user;
ompi_free_list_t sm_first_frags_to_progress; /**< list of first
fragments that are
awaiting resources */
@ -166,10 +179,72 @@ struct mca_btl_sm_component_t {
int sm_fifo_fd; /**< file descriptor corresponding to opened fifo */
opal_thread_t sm_fifo_thread;
#endif
struct mca_btl_sm_t **sm_btls;
struct mca_btl_sm_frag_t **table;
size_t sm_num_btls;
size_t sm_max_btls;
#if OMPI_BTL_SM_HAVE_KNEM
/* Knem capabilities info */
struct knem_cmd_info knem_info;
#endif
/** MCA: should we be using knem or not? neg=try but continue if
not available, 0=don't try, 1=try and fail if not available */
int use_knem;
/** MCA: minimal message size (bytes) to offload on DMA engine
when using knem */
uint32_t knem_dma_min;
/** MCA: how many simultaneous ongoing knem operations to
support */
int knem_max_simultaneous;
/** If we want DMA and DMA is supported, this will be loaded with
KNEM_FLAG_DMA. Otherwise, it'll be 0. */
int knem_dma_flag;
};
typedef struct mca_btl_sm_component_t mca_btl_sm_component_t;
OMPI_MODULE_DECLSPEC extern mca_btl_sm_component_t mca_btl_sm_component;
/**
* SM BTL Interface
*/
struct mca_btl_sm_t {
mca_btl_base_module_t super; /**< base BTL interface */
bool btl_inited; /**< flag indicating if btl has been inited */
mca_btl_base_module_error_cb_fn_t error_cb;
#if OMPI_BTL_SM_HAVE_KNEM
/* File descriptor for knem */
int knem_fd;
/* Array of knem status items for non-blocking knem requests */
knem_status_t *knem_status_array;
/* Array of fragments currently being moved by knem non-blocking
operations */
struct mca_btl_sm_frag_t **knem_frag_array;
/* First free/available location in knem_status_array */
int knem_status_first_avail;
/* First currently-being used location in the knem_status_array */
int knem_status_first_used;
/* Number of status items currently in use */
int knem_status_num_used;
#endif
};
typedef struct mca_btl_sm_t mca_btl_sm_t;
OMPI_MODULE_DECLSPEC extern mca_btl_sm_t mca_btl_sm;
struct btl_sm_pending_send_item_t
{
opal_free_list_item_t super;
@ -314,17 +389,7 @@ extern mca_btl_base_module_t** mca_btl_sm_component_init(
*/
extern int mca_btl_sm_component_progress(void);
/**
* SM BTL Interface
*/
struct mca_btl_sm_t {
mca_btl_base_module_t super; /**< base BTL interface */
bool btl_inited; /**< flag indicating if btl has been inited */
mca_btl_base_module_error_cb_fn_t error_cb;
};
typedef struct mca_btl_sm_t mca_btl_sm_t;
extern mca_btl_sm_t mca_btl_sm;
/**
* Register a callback function that is called on error..
@ -464,6 +529,33 @@ extern int mca_btl_sm_send(
mca_btl_base_tag_t tag
);
#if OMPI_BTL_SM_HAVE_KNEM
/*
* Synchronous knem get
*/
extern int mca_btl_sm_get_sync(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des );
/*
* Asynchronous knem get
*/
extern int mca_btl_sm_get_async(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_btl_base_descriptor_t* des );
extern struct mca_btl_base_descriptor_t* mca_btl_sm_prepare_dst(
struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
struct mca_mpool_base_registration_t* registration,
struct opal_convertor_t* convertor,
uint8_t order,
size_t reserve,
size_t* size,
uint32_t flags);
#endif
/**
* Fault Tolerance Event Notification Function
* @param state Checkpoint Stae

298
ompi/mca/btl/sm/btl_sm_component.c
Просмотреть файл

@ -2,7 +2,7 @@
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@ -43,6 +43,7 @@
#include "opal/event/event.h"
#include "opal/util/output.h"
#include "orte/util/proc_info.h"
#include "orte/util/show_help.h"
#include "orte/runtime/orte_globals.h"
#include "opal/mca/base/mca_base_param.h"
@ -118,6 +119,50 @@ static inline int mca_btl_sm_param_register_int(
int mca_btl_sm_component_open(void)
{
int i;
/* Register an MCA param to indicate whether we have knem support
or not */
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
"have_knem_support", "Whether this component supports the knem Linux kernel module or not",
false, true, OMPI_BTL_SM_HAVE_KNEM, NULL);
if (OMPI_BTL_SM_HAVE_KNEM) {
i = -1;
} else {
i = 0;
}
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
"use_knem",
"Whether knem support is desired or not "
"(negative = try to enable knem support, but continue even if it is not available, 0 = do not enable knem support, positive = try to enable knem support and fail if it is not available)",
false, false, i, &i);
if (OMPI_BTL_SM_HAVE_KNEM) {
mca_btl_sm_component.use_knem = i;
} else {
if (i > 0) {
orte_show_help("help-mpi-btl-sm.txt",
"knem requested but not supported", true,
orte_process_info.nodename);
return OMPI_ERROR;
}
mca_btl_sm_component.use_knem = 0;
}
/* Currently disabling DMA mode by default; it's not clear that
this is useful in all applications and architectures. */
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
"knem_dma_min",
"Minimum message size (in bytes) to use the knem DMA mode; ignored if knem does not support DMA mode (0 = do not use the knem DMA mode)",
false, false, 0, &i);
mca_btl_sm_component.knem_dma_min = (uint32_t) i;
mca_base_param_reg_int(&mca_btl_sm_component.super.btl_version,
"knem_max_simultaneous",
"Max number of simultaneous ongoing knem operations to support (0 = do everything synchronously, which probably gives the best large message latency; >0 means to do all operations asynchronously, which supports better overlap for simultaneous large message sends)",
false, false, 0,
&mca_btl_sm_component.knem_max_simultaneous);
mca_btl_sm_component.sm_max_btls = 1;
/* register SM component parameters */
mca_btl_sm_component.sm_free_list_num =
mca_btl_sm_param_register_int("free_list_num", 8);
@ -135,10 +180,10 @@ int mca_btl_sm_component_open(void)
mca_btl_sm_param_register_int("num_fifos", 1);
/* make sure the number of fifos is a power of 2 */
{
int i = 1;
while ( i < mca_btl_sm_component.nfifos )
i <<= 1;
mca_btl_sm_component.nfifos = i;
int i = 1;
while ( i < mca_btl_sm_component.nfifos )
i <<= 1;
mca_btl_sm_component.nfifos = i;
}
mca_btl_sm_component.fifo_lazy_free =
mca_btl_sm_param_register_int("fifo_lazy_free", 120);
@ -157,15 +202,20 @@ int mca_btl_sm_component_open(void)
mca_btl_sm.super.btl_eager_limit = 4*1024;
mca_btl_sm.super.btl_rndv_eager_limit = 4*1024;
mca_btl_sm.super.btl_max_send_size = 32*1024;
mca_btl_sm.super.btl_rdma_pipeline_send_length = 32*1024;
mca_btl_sm.super.btl_rdma_pipeline_frag_size = 32*1024;
mca_btl_sm.super.btl_min_rdma_pipeline_size = 0;
mca_btl_sm.super.btl_rdma_pipeline_send_length = 64*1024;
mca_btl_sm.super.btl_rdma_pipeline_frag_size = 64*1024;
mca_btl_sm.super.btl_min_rdma_pipeline_size = 64*1024;
mca_btl_sm.super.btl_flags = MCA_BTL_FLAGS_SEND;
mca_btl_sm.super.btl_bandwidth = 900;
mca_btl_sm.super.btl_latency = 100;
#if OMPI_BTL_SM_HAVE_KNEM
if (mca_btl_sm_component.use_knem) {
mca_btl_sm.super.btl_flags |= MCA_BTL_FLAGS_GET;
}
#endif
mca_btl_sm.super.btl_bandwidth = 9000; /* Mbs */
mca_btl_sm.super.btl_latency = 1; /* Microsecs */
mca_btl_base_param_register(&mca_btl_sm_component.super.btl_version,
&mca_btl_sm.super);
&mca_btl_sm.super);
mca_btl_sm_component.max_frag_size = mca_btl_sm.super.btl_max_send_size;
mca_btl_sm_component.eager_limit = mca_btl_sm.super.btl_eager_limit;
@ -173,6 +223,7 @@ int mca_btl_sm_component_open(void)
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_lock, opal_mutex_t);
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_max, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_sm_component.sm_frags_user, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_sm_component.pending_send_fl, opal_free_list_t);
return OMPI_SUCCESS;
}
@ -186,6 +237,22 @@ int mca_btl_sm_component_close(void)
{
int return_value = OMPI_SUCCESS;
#if OMPI_BTL_SM_HAVE_KNEM
if (NULL != mca_btl_sm.knem_frag_array) {
free(mca_btl_sm.knem_frag_array);
mca_btl_sm.knem_frag_array = NULL;
}
if (NULL != mca_btl_sm.knem_status_array) {
munmap(mca_btl_sm.knem_status_array,
mca_btl_sm_component.knem_max_simultaneous);
mca_btl_sm.knem_status_array = NULL;
}
if (-1 != mca_btl_sm.knem_fd) {
close(mca_btl_sm.knem_fd);
mca_btl_sm.knem_fd = -1;
}
#endif
OBJ_DESTRUCT(&mca_btl_sm_component.sm_lock);
/**
* We don't have to destroy the fragment lists. They are allocated
@ -259,6 +326,9 @@ mca_btl_base_module_t** mca_btl_sm_component_init(
bool enable_mpi_threads)
{
mca_btl_base_module_t **btls = NULL;
#if OMPI_BTL_SM_HAVE_KNEM
int rc;
#endif
*num_btls = 0;
@ -278,7 +348,7 @@ mca_btl_base_module_t** mca_btl_sm_component_init(
mca_btl_sm_component.sm_fifo_fd = open(mca_btl_sm_component.sm_fifo_path, O_RDWR);
if(mca_btl_sm_component.sm_fifo_fd < 0) {
opal_output(0, "mca_btl_sm_component_init: open(%s) failed with errno=%d\n",
mca_btl_sm_component.sm_fifo_path, errno);
mca_btl_sm_component.sm_fifo_path, errno);
return NULL;
}
@ -287,6 +357,11 @@ mca_btl_base_module_t** mca_btl_sm_component_init(
opal_thread_start(&mca_btl_sm_component.sm_fifo_thread);
#endif
mca_btl_sm_component.sm_btls = malloc( mca_btl_sm_component.sm_max_btls * sizeof (mca_btl_sm_t *));
if (NULL == mca_btl_sm_component.sm_btls) {
return NULL;
}
/* allocate the Shared Memory BTL */
*num_btls = 1;
btls = (mca_btl_base_module_t**)malloc(sizeof(mca_btl_base_module_t*));
@ -295,17 +370,140 @@ mca_btl_base_module_t** mca_btl_sm_component_init(
}
/* get pointer to the btls */
btls[0] = (mca_btl_base_module_t *)(&(mca_btl_sm));
btls[0] = (mca_btl_base_module_t*)(&(mca_btl_sm));
mca_btl_sm_component.sm_btls[0] = (mca_btl_sm_t*)(&(mca_btl_sm));
/* initialize some PTL data */
/* initialize some BTL data */
/* start with no SM procs */
mca_btl_sm_component.num_smp_procs = 0;
mca_btl_sm_component.my_smp_rank = -1; /* not defined */
mca_btl_sm_component.sm_num_btls = 1;
/* set flag indicating btl not inited */
mca_btl_sm.btl_inited=false;
mca_btl_sm.btl_inited = false;
#if OMPI_BTL_SM_HAVE_KNEM
/* Set knem_status_num_used outside the check for use_knem so that
we can only have to check one thing (knem_status_num_used) in
the progress loop. */
mca_btl_sm.knem_fd = -1;
mca_btl_sm.knem_status_array = NULL;
mca_btl_sm.knem_frag_array = NULL;
mca_btl_sm.knem_status_num_used = 0;
mca_btl_sm.knem_status_first_avail = 0;
mca_btl_sm.knem_status_first_used = 0;
if (0 != mca_btl_sm_component.use_knem) {
/* Open the knem device. Try to print a helpful message if we
fail to open it. */
mca_btl_sm.knem_fd = open("/dev/knem", O_RDWR);
if (mca_btl_sm.knem_fd < 0) {
if (EACCES == errno) {
struct stat sbuf;
if (0 != stat("/dev/knem", &sbuf)) {
sbuf.st_mode = 0;
}
orte_show_help("help-mpi-btl-sm.txt", "knem permission denied",
true, orte_process_info.nodename, sbuf.st_mode);
} else {
orte_show_help("help-mpi-btl-sm.txt", "knem fail open",
true, orte_process_info.nodename, errno,
strerror(errno));
}
goto no_knem;
}
/* Check that the ABI if the kernel module running is the same
as what we were compiled against */
rc = ioctl(mca_btl_sm.knem_fd, KNEM_CMD_GET_INFO,
&mca_btl_sm_component.knem_info);
if (rc < 0) {
orte_show_help("help-mpi-btl-sm.txt", "knem get ABI fail",
true, orte_process_info.nodename, errno,
strerror(errno));
goto no_knem;
}
if (KNEM_ABI_VERSION != mca_btl_sm_component.knem_info.abi) {
orte_show_help("help-mpi-btl-sm.txt", "knem ABI mismatch",
true, orte_process_info.nodename, KNEM_ABI_VERSION,
mca_btl_sm_component.knem_info.abi);
goto no_knem;
}
/* If we want DMA mode and DMA mode is supported, then set
knem_dma_flag to KNEM_FLAG_DMA. */
mca_btl_sm_component.knem_dma_flag = 0;
if (mca_btl_sm_component.knem_dma_min > 0 &&
(mca_btl_sm_component.knem_info.features & KNEM_FEATURE_DMA)) {
mca_btl_sm_component.knem_dma_flag = KNEM_FLAG_DMA;
}
/* Get the array of statuses from knem if max_simultaneous > 0 */
if (mca_btl_sm_component.knem_max_simultaneous > 0) {
mca_btl_sm.knem_status_array = mmap(NULL,
mca_btl_sm_component.knem_max_simultaneous,
(PROT_READ | PROT_WRITE),
MAP_SHARED, mca_btl_sm.knem_fd,
KNEM_STATUS_ARRAY_FILE_OFFSET);
if (MAP_FAILED == mca_btl_sm.knem_status_array) {
orte_show_help("help-mpi-btl-sm.txt", "knem mmap fail",
true, orte_process_info.nodename, errno,
strerror(errno));
goto no_knem;
}
/* The first available status index is 0. Make an empty frag
array. */
mca_btl_sm.knem_frag_array = (mca_btl_sm_frag_t **)
malloc(sizeof(mca_btl_sm_frag_t *) *
mca_btl_sm_component.knem_max_simultaneous);
if (NULL == mca_btl_sm.knem_frag_array) {
orte_show_help("help-mpi-btl-sm.txt", "knem init fail",
true, orte_process_info.nodename, "malloc",
errno, strerror(errno));
goto no_knem;
}
}
}
/* Set the BTL get function pointer if we're supporting KNEM;
choose between synchronous and asynchronous. */
if (mca_btl_sm_component.knem_max_simultaneous > 0) {
mca_btl_sm.super.btl_get = mca_btl_sm_get_async;
} else {
mca_btl_sm.super.btl_get = mca_btl_sm_get_sync;
}
#endif
return btls;
#if OMPI_BTL_SM_HAVE_KNEM
no_knem:
mca_btl_sm.super.btl_flags &= ~MCA_BTL_FLAGS_GET;
if (NULL != mca_btl_sm.knem_frag_array) {
free(mca_btl_sm.knem_frag_array);
mca_btl_sm.knem_frag_array = NULL;
}
if (NULL != mca_btl_sm.knem_status_array) {
munmap(mca_btl_sm.knem_status_array,
mca_btl_sm_component.knem_max_simultaneous);
mca_btl_sm.knem_status_array = NULL;
}
if (-1 != mca_btl_sm.knem_fd) {
close(mca_btl_sm.knem_fd);
mca_btl_sm.knem_fd = -1;
}
/* If "use_knem" is positive, then it's an error if knem support
is not available -- deactivate the sm btl. */
if (mca_btl_sm_component.use_knem > 0) {
return NULL;
}
/* Otherwise, use_knem was 0 (and we didn't get here) or use_knem
was <0, in which case the fact that knem is not available is
not an error. */
return btls;
#endif
}
@ -412,8 +610,8 @@ int mca_btl_sm_component_progress(void)
#if OPAL_ENABLE_DEBUG
if ( FIFO_MAP(peer_smp_rank) != j ) {
opal_output(0, "mca_btl_sm_component_progress: "
"rank %d got %d on FIFO %d, but this sender should send to FIFO %d\n",
my_smp_rank, peer_smp_rank, j, FIFO_MAP(peer_smp_rank));
"rank %d got %d on FIFO %d, but this sender should send to FIFO %d\n",
my_smp_rank, peer_smp_rank, j, FIFO_MAP(peer_smp_rank));
}
#endif
/* recv upcall */
@ -431,15 +629,15 @@ int mca_btl_sm_component_progress(void)
my_smp_rank, peer_smp_rank, hdr->frag, false, true, rc);
break;
}
case MCA_BTL_SM_FRAG_ACK:
case MCA_BTL_SM_FRAG_ACK:
{
int status = (uintptr_t)hdr & MCA_BTL_SM_FRAG_STATUS_MASK;
int btl_ownership;
struct mca_btl_base_endpoint_t* endpoint;
frag = (mca_btl_sm_frag_t *)((char*)((uintptr_t)hdr &
(~(MCA_BTL_SM_FRAG_TYPE_MASK |
MCA_BTL_SM_FRAG_STATUS_MASK))));
(~(MCA_BTL_SM_FRAG_TYPE_MASK |
MCA_BTL_SM_FRAG_STATUS_MASK))));
endpoint = frag->endpoint;
btl_ownership = (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
@ -480,5 +678,63 @@ int mca_btl_sm_component_progress(void)
break;
}
}
#if OMPI_BTL_SM_HAVE_KNEM
/* The sm btl is currently hard-wired for a single module. So
we're not breaking anything here by checking that one module
for knem specifics.
Since knem completes requests in order, we can loop around the
circular status buffer until:
- we find a KNEM_STATUS_PENDING, or
- knem_status_num_used == 0
Note that knem_status_num_used will never be >0 if
component.use_knem<0, so we'll never enter the while loop if
knem is not being used. It will also never be >0 if
max_simultaneous == 0 (because they will all complete
synchronously in _get). However, in order to save a jump
before the return we should test the use_knem here.
*/
if( 0 == mca_btl_sm_component.use_knem ) {
return nevents;
}
while (mca_btl_sm.knem_status_num_used > 0 &&
KNEM_STATUS_PENDING !=
mca_btl_sm.knem_status_array[mca_btl_sm.knem_status_first_used]) {
if (KNEM_STATUS_SUCCESS ==
mca_btl_sm.knem_status_array[mca_btl_sm.knem_status_first_used]) {
int btl_ownership;
/* Handle the completed fragment */
frag =
mca_btl_sm.knem_frag_array[mca_btl_sm.knem_status_first_used];
btl_ownership = (frag->base.des_flags &
MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
if (0 != (MCA_BTL_DES_SEND_ALWAYS_CALLBACK &
frag->base.des_flags)) {
frag->base.des_cbfunc(&mca_btl_sm.super,
frag->endpoint, &frag->base,
OMPI_SUCCESS);
}
if (btl_ownership) {
MCA_BTL_SM_FRAG_RETURN(frag);
}
/* Bump counters, loop around the circular buffer if
necessary */
++nevents;
--mca_btl_sm.knem_status_num_used;
++mca_btl_sm.knem_status_first_used;
if (mca_btl_sm.knem_status_first_used >=
mca_btl_sm_component.knem_max_simultaneous) {
mca_btl_sm.knem_status_first_used = 0;
}
} else {
/* JMS knem fail */
break;
}
}
#endif
return nevents;
}

16
ompi/mca/btl/sm/btl_sm_frag.c
Просмотреть файл

@ -2,13 +2,14 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2005 The University of Tennessee and The University
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -51,6 +52,14 @@ static void mca_btl_sm_frag2_constructor(mca_btl_sm_frag_t* frag)
mca_btl_sm_frag_common_constructor(frag);
}
static void mca_btl_sm_user_constructor(mca_btl_sm_frag_t* frag)
{
frag->size = 0;
frag->my_list = &mca_btl_sm_component.sm_frags_user;
mca_btl_sm_frag_common_constructor(frag);
}
static void mca_btl_sm_frag_destructor(mca_btl_sm_frag_t* frag)
{
}
@ -68,3 +77,8 @@ OBJ_CLASS_INSTANCE(
mca_btl_sm_frag2_constructor,
mca_btl_sm_frag_destructor);
OBJ_CLASS_INSTANCE(
mca_btl_sm_user_t,
mca_btl_base_descriptor_t,
mca_btl_sm_user_constructor,
mca_btl_sm_frag_destructor);

14
ompi/mca/btl/sm/btl_sm_frag.h
Просмотреть файл

@ -2,7 +2,7 @@
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2008 The University of Tennessee and The University
* Copyright (c) 2004-2009 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@ -10,6 +10,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -61,10 +62,13 @@ struct mca_btl_sm_frag_t {
typedef struct mca_btl_sm_frag_t mca_btl_sm_frag_t;
typedef struct mca_btl_sm_frag_t mca_btl_sm_frag1_t;
typedef struct mca_btl_sm_frag_t mca_btl_sm_frag2_t;
typedef struct mca_btl_sm_frag_t mca_btl_sm_user_t;
OBJ_CLASS_DECLARATION(mca_btl_sm_frag_t);
OBJ_CLASS_DECLARATION(mca_btl_sm_frag1_t);
OBJ_CLASS_DECLARATION(mca_btl_sm_frag2_t);
OBJ_CLASS_DECLARATION(mca_btl_sm_user_t);
#define MCA_BTL_SM_FRAG_ALLOC_EAGER(frag, rc) \
{ \
@ -80,6 +84,14 @@ OBJ_CLASS_DECLARATION(mca_btl_sm_frag2_t);
frag = (mca_btl_sm_frag_t*)item; \
}
#define MCA_BTL_SM_FRAG_ALLOC_USER(frag, rc) \
{ \
ompi_free_list_item_t* item; \
OMPI_FREE_LIST_GET(&mca_btl_sm_component.sm_frags_user, item, rc); \
frag = (mca_btl_sm_frag_t*)item; \
}
#define MCA_BTL_SM_FRAG_RETURN(frag) \
{ \
OMPI_FREE_LIST_RETURN(frag->my_list, (ompi_free_list_item_t*)(frag)); \

166
ompi/mca/btl/sm/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,166 @@
# -*- shell-script -*-
#
# Copyright (c) 2009 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# OMPI_CHECK_KNEM(prefix, [action-if-found], [action-if-not-found])
# --------------------------------------------------------
# check if knem support can be found. sets prefix_{CPPFLAGS,
# LDFLAGS, LIBS} as needed and runs action-if-found if there is
# support, otherwise executes action-if-not-found
AC_DEFUN([OMPI_CHECK_KNEM],[
OMPI_VAR_SCOPE_PUSH([ompi_check_knem_happy ompi_check_knem_$1_save_CPPFLAGS ompi_check_knem_dir])
AC_ARG_WITH([knem],
[AC_HELP_STRING([--with-knem(=DIR)],
[Build knem Linux kernel module support, searching for headers in DIR])])
OMPI_CHECK_WITHDIR([knem], [$with_knem], [include/knem_io.h])
ompi_check_knem_$1_save_CPPFLAGS="$CPPFLAGS"
AS_IF([test "$with_knem" != "no"],
[AS_IF([test ! -z "$with_knem" -a "$with_knem" != "yes"],
[ompi_check_knem_dir="$with_knem"])
_OMPI_CHECK_PACKAGE_HEADER([$1],
[knem_io.h],
[$ompi_check_knem_dir],
[ompi_check_knem_happy="yes"],
[ompi_check_knem_happy="no"])],
[ompi_check_knem_happy="no"])
CPPFLAGS="$CPPFLAGS $$1_CPPFLAGS"
# need at least version 0x0000000b
AS_IF([test "$ompi_check_knem_happy" = "yes"],
[AC_CACHE_CHECK([for knem ABI version 0xb or later],
[ompi_cv_knem_version_ok],
[AC_PREPROC_IFELSE(
[AC_LANG_PROGRAM([
#include <knem_io.h>
],[
#if KNEM_ABI_VERSION < 0xc
#error "Version less than 0xc"
#endif
])],
[ompi_cv_knem_version_ok=yes],
[ompi_cv_knem_version_ok=no])])])
CPPFLAGS="$ompi_check_knem_$1_save_CPPFLAGS"
AS_IF([test "$ompi_check_knem_happy" = "yes" -a "$ompi_cv_knem_version_ok" = "yes"],
[$2],
[AS_IF([test ! -z "$with_knem" -a "$with_knem" != "no"],
[AC_MSG_ERROR([KNEM support requested but not found. Aborting])])
$3])
OMPI_VAR_SCOPE_POP
])dnl
# MCA_btl_sm_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_btl_sm_CONFIG],[
OMPI_VAR_SCOPE_PUSH([btl_sm_knem_happy])
OMPI_CHECK_KNEM([btl_sm],
[btl_sm_knem_happy=1],
[btl_sm_knem_happy=0])
AC_DEFINE_UNQUOTED([OMPI_BTL_SM_HAVE_KNEM],
[$btl_sm_knem_happy],
[If knem support can be enabled])
[$1]
# substitute in the things needed to build KNEM
AC_SUBST([btl_sm_CPPFLAGS])
OMPI_VAR_SCOPE_POP
])dnl
# -*- shell-script -*-
#
# Copyright (c) 2009 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# OMPI_CHECK_KNEM(prefix, [action-if-found], [action-if-not-found])
# --------------------------------------------------------
# check if knem support can be found. sets prefix_{CPPFLAGS,
# LDFLAGS, LIBS} as needed and runs action-if-found if there is
# support, otherwise executes action-if-not-found
AC_DEFUN([OMPI_CHECK_KNEM],[
OMPI_VAR_SCOPE_PUSH([ompi_check_knem_happy ompi_check_knem_$1_save_CPPFLAGS ompi_check_knem_dir])
AC_ARG_WITH([knem],
[AC_HELP_STRING([--with-knem(=DIR)],
[Build knem Linux kernel module support, searching for headers in DIR])])
OMPI_CHECK_WITHDIR([knem], [$with_knem], [include/knem_io.h])
ompi_check_knem_$1_save_CPPFLAGS="$CPPFLAGS"
AS_IF([test "$with_knem" != "no"],
[AS_IF([test ! -z "$with_knem" -a "$with_knem" != "yes"],
[ompi_check_knem_dir="$with_knem"])
_OMPI_CHECK_PACKAGE_HEADER([$1],
[knem_io.h],
[$ompi_check_knem_dir],
[ompi_check_knem_happy="yes"],
[ompi_check_knem_happy="no"])],
[ompi_check_knem_happy="no"])
CPPFLAGS="$CPPFLAGS $$1_CPPFLAGS"
# need at least version 0x0000000b
AS_IF([test "$ompi_check_knem_happy" = "yes"],
[AC_CACHE_CHECK([for knem ABI version 0xb or later],
[ompi_cv_knem_version_ok],
[AC_PREPROC_IFELSE(
[AC_LANG_PROGRAM([
#include <knem_io.h>
],[
#if KNEM_ABI_VERSION < 0xc
#error "Version less than 0xc"
#endif
])],
[ompi_cv_knem_version_ok=yes],
[ompi_cv_knem_version_ok=no])])])
CPPFLAGS="$ompi_check_knem_$1_save_CPPFLAGS"
AS_IF([test "$ompi_check_knem_happy" = "yes" -a "$ompi_cv_knem_version_ok" = "yes"],
[$2],
[AS_IF([test ! -z "$with_knem" -a "$with_knem" != "no"],
[AC_MSG_ERROR([KNEM support requested but not found. Aborting])])
$3])
OMPI_VAR_SCOPE_POP
])dnl
# MCA_btl_sm_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_btl_sm_CONFIG],[
OMPI_VAR_SCOPE_PUSH([btl_sm_knem_happy])
OMPI_CHECK_KNEM([btl_sm],
[btl_sm_knem_happy=1],
[btl_sm_knem_happy=0])
AC_DEFINE_UNQUOTED([OMPI_BTL_SM_HAVE_KNEM],
[$btl_sm_knem_happy],
[If knem support can be enabled])
[$1]
# substitute in the things needed to build KNEM
AC_SUBST([btl_sm_CPPFLAGS])
OMPI_VAR_SCOPE_POP
])dnl

78
ompi/mca/btl/sm/help-mpi-btl-sm.txt Обычный файл
Просмотреть файл

@ -0,0 +1,78 @@
# -*- text -*-
#
# Copyright (c) 2004-2009 The University of Tennessee and The University
# of Tennessee Research Foundation. All rights
# reserved.
# Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# This is the US/English help file for Open MPI's shared memory support.
#
[knem requested by not supported]
WARNING: Linux kernel kenm support was requested for the shared memory
(sm) BTL, but it is not supported. Deactivating the shared memory
BTL.
Local host: %s
#
[knem permission denied]
Open MPI failed to open the /dev/knem device due to a permissions
problem. Please check with your system administrator to get the
permissions fixed, or set the btl_sm_use_knem MCA parameter to 0 to
run without /dev/knem support.
Local host: %s
/dev/knem permissions: 0%o
#
[knem fail open]
Open MPI failed to open the /dev/knem device due to a local error.
Please check with your system administrator to get the problem fixed,
or set the btl_sm_use_knem MCA parameter to 0 to run without /dev/knem
support.
Local host: %s
Errno: %d (%s)
#
[knem get ABI fail]
Open MPI failed to retrieve the ABI version from the /dev/knem device
due to a local error. This usually indicates an error in your
/dev/knem installation; please check with your system administrator,
or set the btl_sm_use_knem MCA parameter to 0 to run without /dev/knem
support.
Local host: %s
Errno: %d (%s)
#
[knem ABI mismatch]
Open MPI was compiled with support for one version of the knem kernel
module, but it discovered a different version running in /dev/knem.
Open MPI needs to be installed with support for the same version of
knem as is in the running Linux kernel. Please check with your system
administrator, or set the btl_sm_use_knem MCA parameter to 0 to run
without /dev/knem support.
Local host: %s
Open MPI's knem version: 0x%x
/dev/knem's version: 0x%x
#
[knem mmap fail]
Open MPI failed to map support from the knem Linux kernel module; this
shouldn't happen. Please check with your system administrator, or set
the btl_sm_use_knem MCA parameter to 0 to run without /dev/knem support.
Local host: %s
System call: mmap()
Errno: %d (%s)
#
[knem init error]
Open MPI encountered an error during the knem initialization. Please
check with your system administrator, or set the btl_sm_use_knem MCA
parameter to 0 to run without /dev/knem support.
Local host: %s
System call: %s
Errno: %d (%s)