ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

btl/scif: update for mpool/rcache rewrite

Просмотр исходного кода 
This commit brings the scif btl up to date with changes made on master
to rework the mpool and rcache frameworks.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2016-03-23 16:30:10 -06:00
родитель dec23f3d39
Коммит 478feaf622
4 изменённых файлов: 38 добавлений и 33 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

11
opal/mca/btl/scif/btl_scif.h
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2014 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
@ -16,9 +16,6 @@
#include "opal_config.h"
#include "opal/mca/mpool/mpool.h"
#include "opal/mca/mpool/base/base.h"
#include "opal/mca/mpool/grdma/mpool_grdma.h"
#include "opal/util/output.h"
#include "opal_stdint.h"
#include "opal/util/proc.h"
@ -26,6 +23,8 @@
#include "opal/mca/btl/btl.h"
#include "opal/mca/btl/base/base.h"
#include "opal/mca/btl/base/btl_base_error.h"
#include "opal/mca/rcache/rcache.h"
#include "opal/mca/rcache/base/base.h"
#include <scif.h>
#include <errno.h>
@ -93,6 +92,8 @@ typedef struct mca_btl_scif_module_t {
volatile bool exiting;
bool listening;
mca_rcache_base_module_t *rcache;
} mca_btl_scif_module_t;
typedef struct mca_btl_scif_component_t {
@ -235,7 +236,7 @@ struct mca_btl_scif_registration_handle_t {
typedef struct mca_btl_scif_registration_handle_t mca_btl_scif_registration_handle_t;
typedef struct mca_btl_scif_reg_t {
mca_mpool_base_registration_t base;
mca_rcache_base_registration_t base;
/** per-endpoint btl handles for this registration */
mca_btl_scif_registration_handle_t *handles;
} mca_btl_scif_reg_t;

39
opal/mca/btl/scif/btl_scif_add_procs.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2014 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
@ -17,8 +17,7 @@
#include "btl_scif.h"
#include "btl_scif_frag.h"
static int
mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module);
static int mca_btl_scif_setup_rcache (mca_btl_scif_module_t *scif_module);
static void *mca_btl_scif_connect_accept (void *arg);
int mca_btl_scif_add_procs(struct mca_btl_base_module_t* btl,
@ -48,16 +47,16 @@ int mca_btl_scif_add_procs(struct mca_btl_base_module_t* btl,
procs_on_board++;
}
/* allocate space for the detected peers and setup the mpool */
/* allocate space for the detected peers and setup the rcache */
if (NULL == scif_module->endpoints) {
scif_module->endpoints = calloc (procs_on_board, sizeof (mca_btl_base_endpoint_t));
if (OPAL_UNLIKELY(NULL == scif_module->endpoints)) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
rc = mca_btl_scif_setup_mpools (scif_module);
rc = mca_btl_scif_setup_rcache (scif_module);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
BTL_ERROR(("btl/scif error setting up mpools/free lists"));
BTL_ERROR(("btl/scif error setting up rcache or free lists"));
return rc;
}
}
@ -178,7 +177,7 @@ static int scif_dereg_mem (void *reg_data, mca_mpool_base_registration_t *reg)
}
static int scif_reg_mem (void *reg_data, void *base, size_t size,
mca_mpool_base_registration_t *reg)
mca_rcache_base_registration_t *reg)
{
mca_btl_scif_reg_t *scif_reg = (mca_btl_scif_reg_t *)reg;
int rc = OPAL_SUCCESS;
@ -186,7 +185,7 @@ static int scif_reg_mem (void *reg_data, void *base, size_t size,
scif_reg->handles = calloc (mca_btl_scif_module.endpoint_count, sizeof (scif_reg->handles[0]));
/* intialize all scif offsets to -1 and initialize the pointer back to the mpool registration */
/* intialize all scif offsets to -1 and initialize the pointer back to the rcache registration */
for (i = 0 ; i < mca_btl_scif_module.endpoint_count ; ++i) {
scif_reg->handles[i].btl_handle.scif_offset = -1;
scif_reg->handles[i].btl_handle.scif_base = (intptr_t) base;
@ -211,22 +210,20 @@ static int scif_reg_mem (void *reg_data, void *base, size_t size,
return rc;
}
static int
mca_btl_scif_setup_mpools (mca_btl_scif_module_t *scif_module)
static int mca_btl_scif_setup_rcache (mca_btl_scif_module_t *scif_module)
{
struct mca_mpool_base_resources_t mpool_resources;
mca_rcache_base_resources_t rcache_resources;
int rc;
/* initialize the grdma mpool */
mpool_resources.pool_name = "scif";
mpool_resources.reg_data = (void *) scif_module;
mpool_resources.sizeof_reg = sizeof (mca_btl_scif_reg_t);
mpool_resources.register_mem = scif_reg_mem;
mpool_resources.deregister_mem = scif_dereg_mem;
scif_module->super.btl_mpool =
mca_mpool_base_module_create("grdma", scif_module, &mpool_resources);
if (NULL == scif_module->super.btl_mpool) {
BTL_ERROR(("error creating grdma mpool"));
/* initialize the grdma rcache */
rcache_resources.cache_name = "scif";
rcache_resources.reg_data = (void *) scif_module;
rcache_resources.sizeof_reg = sizeof (mca_btl_scif_reg_t);
rcache_resources.register_mem = scif_reg_mem;
rcache_resources.deregister_mem = scif_dereg_mem;
scif_module->rcache = mca_rcache_base_module_create ("grdma", scif_module, &rcache_resources);
if (NULL == scif_module->rcache) {
BTL_ERROR(("error creating grdma rcache"));
return OPAL_ERROR;
}

4
opal/mca/btl/scif/btl_scif_frag.h
Просмотреть файл

@ -60,8 +60,8 @@ static inline int mca_btl_scif_frag_alloc (mca_btl_base_endpoint_t *ep,
static inline int mca_btl_scif_frag_return (mca_btl_scif_base_frag_t *frag)
{
if (frag->registration) {
frag->endpoint->btl->super.btl_mpool->mpool_deregister(frag->endpoint->btl->super.btl_mpool,
&frag->registration->base);
frag->endpoint->btl->rcache->rcache_deregister (frag->endpoint->btl->rcache,
&frag->registration->base);
frag->registration = NULL;
}

17
opal/mca/btl/scif/btl_scif_module.c
Просмотреть файл

@ -1,6 +1,6 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2014 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
@ -123,6 +123,11 @@ mca_btl_scif_module_finalize (struct mca_btl_base_module_t *btl)
scif_module->endpoints = NULL;
}
if (NULL != scif_module->rcache) {
mca_rcache_base_module_destroy (scif_module->rcache);
scif_module->rcache = NULL;
}
/* close the listening endpoint */
if (mca_btl_scif_module.listening && -1 != mca_btl_scif_module.scif_fd) {
/* wake up the scif thread */
@ -180,6 +185,7 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
mca_btl_base_endpoint_t *endpoint,
void *base, size_t size, uint32_t flags)
{
mca_btl_scif_module_t *scif_module = &mca_btl_scif_module;
mca_btl_scif_reg_t *scif_reg;
int access_flags = flags & MCA_BTL_REG_FLAG_ACCESS_ANY;
int rc;
@ -200,8 +206,8 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
}
}
rc = btl->btl_mpool->mpool_register(btl->btl_mpool, base, size, 0, access_flags,
(mca_mpool_base_registration_t **) &scif_reg);
rc = scif_module->rcache->rcache_register (scif_module->rcache, base, size, 0, access_flags,
(mca_rcache_base_registration_t **) &scif_reg);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
return NULL;
}
@ -210,7 +216,7 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
if ((off_t) -1 == scif_reg->handles[endpoint->id].btl_handle.scif_offset) {
size_t seg_size = (size_t)((uintptr_t) scif_reg->base.bound - (uintptr_t) scif_reg->base.base) + 1;
/* NTH: until we determine a way to pass permissions to the mpool just make all segments
/* NTH: until we determine a way to pass permissions to the rcache just make all segments
* read/write */
scif_reg->handles[endpoint->id].btl_handle.scif_offset =
scif_register (endpoint->scif_epd, scif_reg->base.base, seg_size, 0, SCIF_PROT_READ |
@ -225,9 +231,10 @@ static mca_btl_base_registration_handle_t *mca_btl_scif_register_mem (struct mca
static int mca_btl_scif_deregister_mem (struct mca_btl_base_module_t *btl, mca_btl_base_registration_handle_t *handle)
{
mca_btl_scif_registration_handle_t *scif_handle = (mca_btl_scif_registration_handle_t *) handle;
mca_btl_scif_module_t *scif_module = &mca_btl_scif_module;
mca_btl_scif_reg_t *scif_reg = scif_handle->reg;
btl->btl_mpool->mpool_deregister (btl->btl_mpool, &scif_reg->base);
scif_module->rcache->rcache_deregister (scif_module->rcache, &scif_reg->base);
return OPAL_SUCCESS;
}