ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

opal/common/ucx: Remove unused debugging macro

Просмотр исходного кода 
Will be reintroduced later if needed and after adaptation to the OMPI
infrastructure.

Signed-off-by: Artem Polyakov <artpol84@gmail.com>
Этот коммит содержится в:
Artem Polyakov 2019-01-04 06:11:17 -08:00
родитель 43f16d8796
Коммит 7984d7d997
4 изменённых файлов: 1 добавлений и 231 удалений

14
ompi/mca/osc/ucx/osc_ucx_component.c

@ -552,9 +552,6 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) {
ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module;
int ret;
WPOOL_DBG_OUT(dbg_level, "start, mem = %p lock flag = %d\n",
(void *)module->mem, (int)module->state.lock);
assert(module->lock_count == 0);
assert(opal_list_is_empty(&module->pending_posts) == true);
OBJ_DESTRUCT(&module->outstanding_locks);
@ -562,23 +559,12 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) {
opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_WORKER, 0);
WPOOL_DBG_OUT(dbg_level, "after mem_flush, mem = %p lock flag = %d\n",
(void *)module->mem, (int)module->state.lock);
/*
while (module->state.lock != TARGET_LOCK_UNLOCKED) {
ucp_worker_progress(mca_osc_ucx_component.wpool->recv_worker);
}
*/
ret = module->comm->c_coll->coll_barrier(module->comm,
module->comm->c_coll->coll_barrier_module);
if (ret != OMPI_SUCCESS) {
return ret;
}
WPOOL_DBG_OUT(dbg_level, "after barrier, mem = %p\n", (void *)module->mem);
free(module->addrs);
free(module->state_addrs);

11
ompi/mca/osc/ucx/osc_ucx_passive_target.c

@ -33,9 +33,6 @@ static inline int start_shared(ompi_osc_ucx_module_t *module, int target) {
return ret;
}
WPOOL_DBG_OUT(dbg_level, "start_shared: after fadd, result_value = %d",
(int)result_value);
assert((int64_t)result_value >= 0);
if (result_value >= TARGET_LOCK_EXCLUSIVE) {
ret = opal_common_ucx_wpmem_post(module->state_mem,
@ -222,8 +219,6 @@ int ompi_osc_ucx_unlock_all(struct ompi_win_t *win) {
int comm_size = ompi_comm_size(module->comm);
int ret = OMPI_SUCCESS;
WPOOL_DBG_OUT(dbg_level, "start, mem = %p\n", (void *)module->mem);
if (module->epoch_type.access != PASSIVE_ALL_EPOCH) {
return OMPI_ERR_RMA_SYNC;
}
@ -235,9 +230,6 @@ int ompi_osc_ucx_unlock_all(struct ompi_win_t *win) {
return ret;
}
WPOOL_DBG_OUT(dbg_level, "done flushing: mem = %p\n",
(void *)module->mem);
if (!module->lock_all_is_nocheck) {
int i;
for (i = 0; i < comm_size; i++) {
@ -247,9 +239,6 @@ int ompi_osc_ucx_unlock_all(struct ompi_win_t *win) {
module->epoch_type.access = NONE_EPOCH;
WPOOL_DBG_OUT(dbg_level, "fini: mem = %p\n",
(void *)module->mem);
return ret;
}

125
opal/mca/common/ucx/common_ucx_wpool.c

@ -66,8 +66,6 @@ _winfo_create(opal_common_ucx_wpool_t *wpool)
winfo->global_inflight_ops = 0;
winfo->inflight_req = UCS_OK;
WPOOL_DBG_OUT(_dbg_winfo, "winfo = %p, worker = %p\n",
(void*)winfo, (void *)winfo->worker);
return winfo;
release_worker:
@ -101,15 +99,11 @@ _winfo_reset(opal_common_ucx_winfo_t *winfo)
winfo->endpoints = NULL;
winfo->comm_size = 0;
winfo->released = 0;
WPOOL_DBG_OUT(_dbg_winfo, "winfo = %p, worker = %p\n",
(void*)winfo, (void*)winfo->worker);
}
static void
_winfo_release(opal_common_ucx_winfo_t *winfo)
{
WPOOL_DBG_OUT(_dbg_winfo, "winfo = %p, worker = %p\n",
(void*)winfo, (void *)winfo->worker);
OBJ_DESTRUCT(&winfo->mutex);
ucp_worker_destroy(winfo->worker);
free(winfo);
@ -125,7 +119,6 @@ opal_common_ucx_wpool_allocate(void)
opal_common_ucx_wpool_t *ptr = calloc(1, sizeof(opal_common_ucx_wpool_t));
ptr->refcnt = 0;
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p\n", (void *)ptr);
return ptr;
}
@ -133,7 +126,6 @@ OPAL_DECLSPEC void
opal_common_ucx_wpool_free(opal_common_ucx_wpool_t *wpool)
{
assert(wpool->refcnt == 0);
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p\n", (void *)wpool);
free(wpool);
}
@ -148,8 +140,6 @@ opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool,
int rc = OPAL_SUCCESS;
wpool->refcnt++;
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, refctnrt = %d\n",
(void *)wpool, wpool->refcnt);
if (1 < wpool->refcnt) {
return rc;
@ -213,7 +203,6 @@ opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool,
pthread_key_create(&wpool->tls_key, _tlocal_cleanup);
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Done\n", (void *)wpool);
return rc;
err_wpool_add:
@ -233,11 +222,8 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
{
_tlocal_table_t *tls_item = NULL, *tls_next;
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Start\n", (void *)wpool);
wpool->refcnt--;
if (wpool->refcnt > 0) {
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Still in use\n", (void *)wpool);
return;
}
@ -250,8 +236,6 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
_tlocal_table_t) {
opal_list_remove_item(&wpool->tls_list, &tls_item->super);
_common_ucx_tls_cleanup(tls_item);
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Cleanup TLS = %p\n",
(void *)wpool, (void*)tls_item);
}
OBJ_DESTRUCT(&wpool->tls_list);
@ -265,8 +249,6 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
OPAL_LIST_FOREACH_SAFE(item, next, &wpool->idle_workers,
_winfo_list_item_t) {
opal_list_remove_item(&wpool->idle_workers, &item->super);
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Cleanup idle winfo = %p\n",
(void *)wpool, (void*)item->ptr);
_winfo_release(item->ptr);
OBJ_RELEASE(item);
}
@ -282,8 +264,6 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
opal_list_remove_item(&wpool->active_workers, &item->super);
_winfo_reset(item->ptr);
_winfo_release(item->ptr);
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Cleanup active winfo = %p\n",
(void *)wpool, (void*)item->ptr);
OBJ_RELEASE(item);
}
}
@ -291,7 +271,6 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
OBJ_DESTRUCT(&wpool->mutex);
ucp_cleanup(wpool->ucp_ctx);
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Done\n", (void *)wpool);
return;
}
@ -329,9 +308,6 @@ _wpool_list_put(opal_common_ucx_wpool_t *wpool, opal_list_t *list,
{
_winfo_list_item_t *item;
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, winfo = %p\n",
(void *)wpool, (void *)winfo);
item = OBJ_NEW(_winfo_list_item_t);
if (NULL == item) {
MCA_COMMON_UCX_ERROR("Cannot allocate memory for winfo list item");
@ -363,8 +339,6 @@ _wpool_list_get(opal_common_ucx_wpool_t *wpool, opal_list_t *list)
winfo = item->ptr;
OBJ_RELEASE(item);
}
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, winfo = %p\n",
(void *)wpool, (void *)winfo);
return winfo;
}
@ -381,9 +355,6 @@ _wpool_get_idle(opal_common_ucx_wpool_t *wpool, size_t comm_size)
}
}
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, winfo = %p\n",
(void *)wpool, (void *)winfo);
winfo->endpoints = calloc(comm_size, sizeof(ucp_ep_h));
winfo->inflight_ops = calloc(comm_size, sizeof(short));
winfo->comm_size = comm_size;
@ -393,8 +364,6 @@ _wpool_get_idle(opal_common_ucx_wpool_t *wpool, size_t comm_size)
static int
_wpool_add_active(opal_common_ucx_wpool_t *wpool, opal_common_ucx_winfo_t *winfo)
{
WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, winfo = %p\n",
(void *)wpool, (void *)winfo);
return _wpool_list_put(wpool, &wpool->active_workers, winfo);
}
@ -411,8 +380,6 @@ opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size,
opal_common_ucx_ctx_t *ctx = calloc(1, sizeof(*ctx));
int ret = OPAL_SUCCESS;
WPOOL_DBG_OUT(_dbg_ctx, "ctx_create: ctx = %p\n", (void*)ctx);
OBJ_CONSTRUCT(&ctx->mutex, opal_recursive_mutex_t);
OBJ_CONSTRUCT(&ctx->tls_workers, opal_list_t);
ctx->released = 0;
@ -430,8 +397,6 @@ opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size,
}
(*ctx_ptr) = ctx;
WPOOL_DBG_OUT(_dbg_ctx,"wpool = %p, ctx = %p\n",
(void *)wpool, (void *)(*ctx_ptr));
return ret;
error:
OBJ_DESTRUCT(&ctx->mutex);
@ -446,8 +411,6 @@ opal_common_ucx_wpctx_release(opal_common_ucx_ctx_t *ctx)
{
int my_refcntr = -1;
WPOOL_DBG_OUT(_dbg_ctx, "ctx = %p\n", (void *)ctx);
/* Application is expected to guarantee that no operation
* is performed on the context that is being released */
@ -477,7 +440,6 @@ _common_ucx_wpctx_free(opal_common_ucx_ctx_t *ctx)
free(ctx->recv_worker_displs);
OBJ_DESTRUCT(&ctx->mutex);
OBJ_DESTRUCT(&ctx->tls_workers);
WPOOL_DBG_OUT(_dbg_ctx, "ctx = %p\n", (void *)ctx);
free(ctx);
}
@ -499,8 +461,6 @@ _common_ucx_wpctx_append(opal_common_ucx_ctx_t *ctx,
opal_list_append(&ctx->tls_workers, &item->super);
opal_mutex_unlock(&ctx->mutex);
WPOOL_DBG_OUT(_dbg_ctx, "ctx = %p, winfo = %p\n",
(void *)ctx, (void *)winfo);
return OPAL_SUCCESS;
}
@ -541,7 +501,6 @@ _common_ucx_wpctx_remove(opal_common_ucx_ctx_t *ctx,
* We can safely release communication context structure */
_common_ucx_wpctx_free(ctx);
}
WPOOL_DBG_OUT(_dbg_ctx, "ctx = %p\n", (void *)ctx);
return;
}
@ -565,8 +524,6 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
ucs_status_t status;
int ret = OPAL_SUCCESS;
WPOOL_DBG_OUT(_dbg_mem, "for ctx = %p\n", (void *)ctx);
mem->released = 0;
mem->refcntr = 1; /* application holding this memory handler */
mem->ctx = ctx;
@ -579,8 +536,6 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
MCA_COMMON_UCX_VERBOSE(1, "_comm_ucx_mem_map failed: %d", ret);
goto error_mem_map;
}
WPOOL_DBG_OUT(_dbg_mem, "\tbase = %p, memh = %p\n",
(void *)(*mem_base), (void *)(mem->memh));
status = ucp_rkey_pack(ctx->wpool->ucp_ctx, mem->memh,
&rkey_addr, &rkey_addr_len);
@ -589,12 +544,9 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
ret = OPAL_ERROR;
goto error_rkey_pack;
}
WPOOL_DBG_OUT(_dbg_mem, "\trkey_addr = %p, rkey_addr_len = %d\n",
(void *)rkey_addr, (int)rkey_addr_len);
ret = exchange_func(rkey_addr, rkey_addr_len,
&mem->mem_addrs, &mem->mem_displs, exchange_metadata);
WPOOL_DBG_OUT(_dbg_mem, "\tcomplete exchange");
if (ret != OPAL_SUCCESS) {
goto error_rkey_pack;
}
@ -607,7 +559,6 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
(*my_mem_addr) = rkey_addr;
(*my_mem_addr_size) = rkey_addr_len;
WPOOL_DBG_OUT(_dbg_mem, "mem = %p. Done\n", (void *)mem);
return ret;
error_rkey_pack:
@ -623,8 +574,6 @@ opal_common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem)
{
int my_refcntr = -1;
WPOOL_DBG_OUT(_dbg_mem, "mem = %p\n", (void *)mem);
/* Mark that this memory handler has been called */
mem->released = 1;
@ -650,8 +599,6 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
ucs_status_t status;
int ret = OPAL_SUCCESS;
WPOOL_DBG_OUT(_dbg_mem, "wpool = %p\n", (void *)wpool);
memset(&mem_params, 0, sizeof(ucp_mem_map_params_t));
mem_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS |
UCP_MEM_MAP_PARAM_FIELD_LENGTH |
@ -670,7 +617,6 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
ret = OPAL_ERROR;
return ret;
}
WPOOL_DBG_OUT(_dbg_mem, "\tmemh = %p\n", (void *)(*memh_ptr));
mem_attrs.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS | UCP_MEM_ATTR_FIELD_LENGTH;
status = ucp_mem_query((*memh_ptr), &mem_attrs);
@ -679,7 +625,6 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
ret = OPAL_ERROR;
goto error;
}
WPOOL_DBG_OUT(_dbg_mem, "\tmemh = %p\n", (void *)(*memh_ptr));
assert(mem_attrs.length >= size);
if (mem_type != OPAL_COMMON_UCX_MEM_ALLOCATE_MAP) {
@ -688,8 +633,6 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
(*base) = mem_attrs.address;
}
WPOOL_DBG_OUT(_dbg_mem, "\twpool = %p, addr = %p size = %d memh = %p\n",
(void *)wpool, (void *)(*base), (int)size, (void *)(*memh_ptr));
return ret;
error:
ucp_mem_unmap(wpool->ucp_ctx, (*memh_ptr));
@ -702,7 +645,6 @@ static void _common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem)
free(mem->mem_addrs);
free(mem->mem_displs);
ucp_mem_unmap(mem->ctx->wpool->ucp_ctx, mem->memh);
WPOOL_DBG_OUT(_dbg_mem, "mem = %p\n", (void *)mem);
free(mem);
}
@ -711,8 +653,6 @@ _common_ucx_wpmem_signup(opal_common_ucx_wpmem_t *mem)
{
/* Increment the reference counter */
OPAL_ATOMIC_ADD_FETCH32(&mem->refcntr, 1);
WPOOL_DBG_OUT(_dbg_mem, "mem = %p\n", (void *)mem);
return OPAL_SUCCESS;
}
@ -737,7 +677,6 @@ _common_ucx_mem_signout(opal_common_ucx_wpmem_t *mem)
_common_ucx_wpmem_free(mem);
}
WPOOL_DBG_OUT(_dbg_mem, "mem = %p\n", (void *)mem);
return;
}
@ -777,9 +716,6 @@ static _tlocal_table_t* _common_ucx_tls_init(opal_common_ucx_wpool_t *wpool)
pthread_setspecific(wpool->tls_key, tls);
WPOOL_DBG_OUT(_dbg_tls, "tls = %p, wpool = %p\n",
(void *)tls, (void*)wpool);
return tls;
}
@ -789,8 +725,6 @@ _tlocal_get_tls(opal_common_ucx_wpool_t *wpool){
if( OPAL_UNLIKELY(NULL == tls) ) {
tls = _common_ucx_tls_init(wpool);
}
WPOOL_DBG_OUT(_dbg_tls, "tls = %p, wpool = %p\n",
(void *)tls, (void*)wpool);
return tls;
}
@ -805,9 +739,6 @@ static void _tlocal_cleanup(void *arg)
}
wpool = tls->wpool;
WPOOL_DBG_OUT(_dbg_tls, "tls = %p, wpool = %p\n",
(void *)tls, (void*)wpool);
/* 1. Remove us from tls_list */
tls->wpool = wpool;
opal_mutex_lock(&wpool->mutex);
@ -848,8 +779,6 @@ static void _common_ucx_tls_cleanup(_tlocal_table_t *tls)
pthread_setspecific(tls->wpool->tls_key, NULL);
WPOOL_DBG_OUT(_dbg_tls, "tls = %p, wpool = %p\n",
(void *)tls, (void*)tls->wpool);
OBJ_RELEASE(tls);
return;
}
@ -868,7 +797,6 @@ _tlocal_tls_ctxtbl_extend(_tlocal_table_t *tbl, size_t append)
}
tbl->ctx_tbl_size = newsize;
WPOOL_DBG_OUT(_dbg_tls, "new size = %d\n", (int)newsize);
return OPAL_SUCCESS;
}
@ -886,7 +814,6 @@ _tlocal_tls_memtbl_extend(_tlocal_table_t *tbl, size_t append)
}
}
tbl->mem_tbl_size = newsize;
WPOOL_DBG_OUT(_dbg_tls, "new size = %d\n", (int)newsize);
return OPAL_SUCCESS;
}
@ -900,7 +827,6 @@ _tlocal_ctx_search(_tlocal_table_t *tls, opal_common_ucx_ctx_t *ctx)
return tls->ctx_tbl[i];
}
}
WPOOL_DBG_OUT(_dbg_tls, "tls = %p, ctx = %p\n", (void *)tls, (void*)ctx);
return NULL;
}
@ -920,10 +846,6 @@ _tlocal_ctx_record_cleanup(_tlocal_ctx_t *ctx_rec)
* delayed cleanup */
_common_ucx_wpctx_remove(ctx_rec->gctx, ctx_rec->winfo);
WPOOL_DBG_OUT(_dbg_tls, "wpool = %p, winfo = %p, worker = %p\n",
(void*)ctx_rec->gctx->wpool, (void*)ctx_rec->winfo,
(void*)ctx_rec->winfo->worker);
/* Erase the record so it can be reused */
memset(ctx_rec, 0, sizeof(*ctx_rec));
@ -987,10 +909,6 @@ _tlocal_add_ctx(_tlocal_table_t *tls, opal_common_ucx_ctx_t *ctx)
return NULL;
}
WPOOL_DBG_OUT(_dbg_tls || _dbg_ctx, "tls = %p, ctx_rec = %p, winfo = %p\n",
(void *)tls, (void *)&tls->ctx_tbl[free_idx],
(void *)tls->ctx_tbl[free_idx]->winfo);
/* All good - return the record */
return tls->ctx_tbl[free_idx];
}
@ -1015,8 +933,6 @@ static int _tlocal_ctx_connect(_tlocal_ctx_t *ctx_rec, int target)
MCA_COMMON_UCX_VERBOSE(1, "ucp_ep_create failed: %d", status);
return OPAL_ERROR;
}
WPOOL_DBG_OUT(_dbg_tls || _dbg_ctx, "worker = %p ep = %p\n",
(void *)winfo->worker, (void *)winfo->endpoints[target]);
opal_mutex_unlock(&winfo->mutex);
return OPAL_SUCCESS;
}
@ -1027,8 +943,6 @@ static inline _tlocal_mem_t *
_tlocal_search_mem(_tlocal_table_t *tls, opal_common_ucx_wpmem_t *gmem)
{
size_t i;
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p mem = %p\n",
(void *)tls, (void*)gmem);
for(i=0; i<tls->mem_tbl_size; i++) {
if( tls->mem_tbl[i]->gmem == gmem){
return tls->mem_tbl[i];
@ -1041,14 +955,10 @@ static void
_tlocal_mem_record_cleanup(_tlocal_mem_t *mem_rec)
{
size_t i;
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "record=%p, is_freed = %d\n",
(void *)mem_rec, mem_rec->gmem->released);
for(i = 0; i < mem_rec->gmem->ctx->comm_size; i++) {
if (mem_rec->mem->rkeys[i]) {
ucp_rkey_destroy(mem_rec->mem->rkeys[i]);
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "rkey_entry = %p\n",
(void *)mem_rec->mem->rkeys[i]);
}
}
free(mem_rec->mem->rkeys);
@ -1057,8 +967,6 @@ _tlocal_mem_record_cleanup(_tlocal_mem_t *mem_rec)
* This may result in context release as we are using
* delayed cleanup */
_common_ucx_mem_signout(mem_rec->gmem);
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "gmem = %p mem_rec = %p\n",
(void *)mem_rec->gmem, (void *)mem_rec);
/* Release fast-path pointers */
if (NULL != mem_rec->mem_tls_ptr) {
@ -1103,7 +1011,6 @@ static _tlocal_mem_t *_tlocal_add_mem(_tlocal_table_t *tls,
//TODO: error out
return NULL;
}
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p\n", (void *)tls);
}
tls->mem_tbl[free_idx]->gmem = mem;
@ -1114,8 +1021,6 @@ static _tlocal_mem_t *_tlocal_add_mem(_tlocal_table_t *tls,
// TODO: act accordingly - cleanup
return NULL;
}
WPOOL_DBG_OUT("tls = %p, ctx = %p\n",
(void *)tls, (void*)mem->ctx);
tls->mem_tbl[free_idx]->ctx_rec = ctx_rec;
OPAL_ATOMIC_ADD_FETCH32(&ctx_rec->refcnt, 1);
@ -1142,9 +1047,6 @@ static _tlocal_mem_t *_tlocal_add_mem(_tlocal_table_t *tls,
// TODO: error handling
return NULL;
}
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem,
"mem = %p, mem_tbl_entry = %p\n",
(void *)mem, (void *)tls->mem_tbl[free_idx]);
return tls->mem_tbl[free_idx];
}
@ -1163,8 +1065,7 @@ _tlocal_mem_create_rkey(_tlocal_mem_t *mem_rec, ucp_ep_h ep, int target)
MCA_COMMON_UCX_VERBOSE(1, "ucp_ep_rkey_unpack failed: %d", status);
return OPAL_ERROR;
}
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "mem_rec = %p ep = %p target = %d\n",
(void *)mem_rec, (void *)ep, target);
return OPAL_SUCCESS;
}
@ -1181,24 +1082,17 @@ opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target)
int rc = OPAL_SUCCESS;
tls = _tlocal_get_tls(mem->ctx->wpool);
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p\n",(void*)tls);
/* Obtain the worker structure */
ctx_rec = _tlocal_ctx_search(tls, mem->ctx);
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "ctx_id = %p, ctx_rec=%p\n",
(void*)mem->ctx, (void *)ctx_rec);
if (OPAL_UNLIKELY(NULL == ctx_rec)) {
ctx_rec = _tlocal_add_ctx(tls, mem->ctx);
if (NULL == ctx_rec) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
WPOOL_DBG_OUT("_tlocal_fetch(after _tlocal_add_ctx): tls = %p ctx = %p\n",
(void *)tls, (void *)mem->ctx);
}
winfo = ctx_rec->winfo;
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "winfo = %p ctx=%p\n",
(void *)winfo, (void *)mem->ctx);
/* Obtain the endpoint */
if (OPAL_UNLIKELY(NULL == winfo->endpoints[target])) {
@ -1206,26 +1100,18 @@ opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target)
if (rc != OPAL_SUCCESS) {
return rc;
}
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "ctx_rec = %p target = %d\n",
(void *)ctx_rec, target);
}
ep = winfo->endpoints[target];
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "ep = %p\n", (void *)ep);
/* Obtain the memory region info */
mem_rec = _tlocal_search_mem(tls, mem);
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p mem_rec = %p mem_id = %p\n",
(void *)tls, (void *)mem_rec, (void*)mem);
if (OPAL_UNLIKELY(mem_rec == NULL)) {
mem_rec = _tlocal_add_mem(tls, mem);
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p mem = %p\n",
(void *)tls, (void *)mem);
if (NULL == mem_rec) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
}
mem_info = mem_rec->mem;
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "mem_info = %p\n", (void *)mem_info);
/* Obtain the rkey */
if (OPAL_UNLIKELY(NULL == mem_info->rkeys[target])) {
@ -1234,7 +1120,6 @@ opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target)
if (rc) {
return rc;
}
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "creating rkey ...\n");
}
return OPAL_SUCCESS;
@ -1293,9 +1178,6 @@ opal_common_ucx_wpmem_flush(opal_common_ucx_wpmem_t *mem,
opal_common_ucx_ctx_t *ctx = mem->ctx;
int rc = OPAL_SUCCESS;
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "mem = %p, target = %d\n",
(void *)mem, target);
opal_mutex_lock(&ctx->mutex);
OPAL_LIST_FOREACH(item, &ctx->tls_workers, _ctx_record_list_item_t) {
@ -1323,11 +1205,6 @@ opal_common_ucx_wpmem_flush(opal_common_ucx_wpmem_t *mem,
rc);
rc = OPAL_ERROR;
}
WPOOL_DBG_OUT(_dbg_tls || _dbg_mem,
"target = %d, ep = %p worker = %p\n",
(int)target,
(void *)item->ptr->endpoints[target],
(void *)item->ptr->worker);
}
opal_mutex_unlock(&ctx->mutex);

82
opal/mca/common/ucx/common_ucx_wpool.h

@ -127,68 +127,6 @@ typedef int (*opal_common_ucx_exchange_func_t)(void *my_info, size_t my_info_len
char **recv_info, int **disps,
void *metadata);
/* For developer use only */
//#define OPAL_COMMON_UCX_WPOOL_DBG
#ifdef OPAL_COMMON_UCX_WPOOL_DBG
extern __thread FILE *tls_pf;
extern __thread int initialized;
#include <unistd.h>
#include <sys/syscall.h>
#include <time.h>
#include <sys/time.h>
static int _dbg_winfo = 0;
static int _dbg_wpool = 0;
static int _dbg_ctx = 0;
static int _dbg_mem = 0;
static int _dbg_tls = 0;
static inline void opal_common_ucx_wpool_dbg_init(void)
{
if( !initialized ) {
int tid = syscall(__NR_gettid);
char hname[128];
gethostname(hname, 127);
char fname[128];
sprintf(fname, "%s.%d.log", hname, tid);
tls_pf = fopen(fname, "w");
initialized = 1;
// Create issusion that they are used to avoid compiler warnings
(void)_dbg_ctx;
(void)_dbg_mem;
(void)_dbg_tls;
(void)_dbg_winfo;
(void)_dbg_wpool;
}
}
#define WPOOL_DBG_OUT(level, ...) \
{ \
struct timeval start_; \
time_t nowtime_; \
struct tm *nowtm_; \
char tmbuf_[64]; \
gettimeofday(&start_, NULL); \
nowtime_ = start_.tv_sec; \
nowtm_ = localtime(&nowtime_); \
strftime(tmbuf_, sizeof(tmbuf_), \
"%H:%M:%S", nowtm_); \
opal_common_ucx_wpool_dbg_init(); \
if (level) { \
fprintf(tls_pf, "[%s.%06ld] %s:", \
tmbuf_, start_.tv_usec, \
__func__); \
} \
fprintf(tls_pf, __VA_ARGS__); \
}
#else
#define WPOOL_DBG_OUT(...)
#endif
/* Manage Worker Pool (wpool) */
OPAL_DECLSPEC opal_common_ucx_wpool_t * opal_common_ucx_wpool_allocate(void);
@ -374,8 +312,6 @@ opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem, opal_common_ucx_op_t
MCA_COMMON_UCX_VERBOSE(1, "tlocal_fetch failed: %d", rc);
return rc;
}
WPOOL_DBG_OUT(_dbg_mem, "mem = %p, ep = %p, rkey = %p, winfo = %p\n",
(void *)mem, (void *)ep, (void *)rkey, (void *)winfo);
/* Perform the operation */
opal_mutex_lock(&winfo->mutex);
@ -393,9 +329,6 @@ opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem, opal_common_ucx_op_t
if (OPAL_UNLIKELY(status != UCS_OK && status != UCS_INPROGRESS)) {
MCA_COMMON_UCX_ERROR("%s failed: %d", called_func, status);
rc = OPAL_ERROR;
} else {
WPOOL_DBG_OUT(_dbg_mem,"ep = %p, rkey = %p\n",
(void *)ep, (void *)rkey);
}
rc = _periodical_flush_nb(mem, winfo, target);
@ -426,8 +359,6 @@ opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare,
MCA_COMMON_UCX_ERROR("opal_common_ucx_tlocal_fetch failed: %d", rc);
return rc;
}
WPOOL_DBG_OUT("mem = %p, ep = %p, rkey = %p, winfo = %p\n",
(void *)mem, (void *)ep, (void *)rkey, (void *)winfo);
/* Perform the operation */
opal_mutex_lock(&winfo->mutex);
@ -438,9 +369,6 @@ opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare,
if (OPAL_UNLIKELY(status != UCS_OK)) {
MCA_COMMON_UCX_ERROR("opal_common_ucx_atomic_cswap failed: %d", status);
rc = OPAL_ERROR;
} else {
WPOOL_DBG_OUT(_dbg_mem, "ep = %p, rkey = %p\n",
(void *)ep, (void *)rkey);
}
rc = _periodical_flush_nb(mem, winfo, target);
@ -470,8 +398,6 @@ opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem, ucp_atomic_post_op_t op
MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc);
return rc;
}
WPOOL_DBG_OUT(_dbg_mem, "mem = %p, ep = %p, rkey = %p, winfo = %p\n",
(void *)mem, (void *)ep, (void *)rkey, (void *)winfo);
/* Perform the operation */
opal_mutex_lock(&winfo->mutex);
@ -480,9 +406,6 @@ opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem, ucp_atomic_post_op_t op
if (OPAL_UNLIKELY(status != UCS_OK)) {
MCA_COMMON_UCX_ERROR("ucp_atomic_post failed: %d", status);
rc = OPAL_ERROR;
} else {
WPOOL_DBG_OUT(_dbg_mem, "ep = %p, rkey = %p\n",
(void *)ep, (void *)rkey);
}
rc = _periodical_flush_nb(mem, winfo, target);
@ -512,8 +435,6 @@ opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem,
MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc);
return rc;
}
WPOOL_DBG_OUT(_dbg_mem, "mem = %p, ep = %p, rkey = %p, winfo = %p\n",
(void *)mem, (void *)ep, (void *)rkey, (void *)winfo);
/* Perform the operation */
opal_mutex_lock(&winfo->mutex);
@ -524,9 +445,6 @@ opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem,
if (OPAL_UNLIKELY(status != UCS_OK)) {
MCA_COMMON_UCX_ERROR("ucp_atomic_cswap64 failed: %d", status);
rc = OPAL_ERROR;
} else {
WPOOL_DBG_OUT(_dbg_mem, "ep = %p, rkey = %p\n",
(void *)ep, (void *)rkey);
}
rc = _periodical_flush_nb(mem, winfo, target);