diff --git a/ompi/mca/osc/ucx/osc_ucx_component.c b/ompi/mca/osc/ucx/osc_ucx_component.c index 207b649ebb..d96a694dfe 100644 --- a/ompi/mca/osc/ucx/osc_ucx_component.c +++ b/ompi/mca/osc/ucx/osc_ucx_component.c @@ -552,9 +552,6 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) { ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module; int ret; - WPOOL_DBG_OUT(dbg_level, "start, mem = %p lock flag = %d\n", - (void *)module->mem, (int)module->state.lock); - assert(module->lock_count == 0); assert(opal_list_is_empty(&module->pending_posts) == true); OBJ_DESTRUCT(&module->outstanding_locks); @@ -562,23 +559,12 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) { opal_common_ucx_wpmem_flush(module->mem, OPAL_COMMON_UCX_SCOPE_WORKER, 0); - WPOOL_DBG_OUT(dbg_level, "after mem_flush, mem = %p lock flag = %d\n", - (void *)module->mem, (int)module->state.lock); - - /* - while (module->state.lock != TARGET_LOCK_UNLOCKED) { - ucp_worker_progress(mca_osc_ucx_component.wpool->recv_worker); - } - */ - ret = module->comm->c_coll->coll_barrier(module->comm, module->comm->c_coll->coll_barrier_module); if (ret != OMPI_SUCCESS) { return ret; } - WPOOL_DBG_OUT(dbg_level, "after barrier, mem = %p\n", (void *)module->mem); - free(module->addrs); free(module->state_addrs); diff --git a/ompi/mca/osc/ucx/osc_ucx_passive_target.c b/ompi/mca/osc/ucx/osc_ucx_passive_target.c index 0c1e8bddfd..1f5e73b225 100644 --- a/ompi/mca/osc/ucx/osc_ucx_passive_target.c +++ b/ompi/mca/osc/ucx/osc_ucx_passive_target.c @@ -33,9 +33,6 @@ static inline int start_shared(ompi_osc_ucx_module_t *module, int target) { return ret; } - WPOOL_DBG_OUT(dbg_level, "start_shared: after fadd, result_value = %d", - (int)result_value); - assert((int64_t)result_value >= 0); if (result_value >= TARGET_LOCK_EXCLUSIVE) { ret = opal_common_ucx_wpmem_post(module->state_mem, @@ -222,8 +219,6 @@ int ompi_osc_ucx_unlock_all(struct ompi_win_t *win) { int comm_size = ompi_comm_size(module->comm); int ret = OMPI_SUCCESS; - WPOOL_DBG_OUT(dbg_level, "start, mem = %p\n", (void *)module->mem); - if (module->epoch_type.access != PASSIVE_ALL_EPOCH) { return OMPI_ERR_RMA_SYNC; } @@ -235,9 +230,6 @@ int ompi_osc_ucx_unlock_all(struct ompi_win_t *win) { return ret; } - WPOOL_DBG_OUT(dbg_level, "done flushing: mem = %p\n", - (void *)module->mem); - if (!module->lock_all_is_nocheck) { int i; for (i = 0; i < comm_size; i++) { @@ -247,9 +239,6 @@ int ompi_osc_ucx_unlock_all(struct ompi_win_t *win) { module->epoch_type.access = NONE_EPOCH; - WPOOL_DBG_OUT(dbg_level, "fini: mem = %p\n", - (void *)module->mem); - return ret; } diff --git a/opal/mca/common/ucx/common_ucx_wpool.c b/opal/mca/common/ucx/common_ucx_wpool.c index c9703d2866..2299a7cd0e 100644 --- a/opal/mca/common/ucx/common_ucx_wpool.c +++ b/opal/mca/common/ucx/common_ucx_wpool.c @@ -66,8 +66,6 @@ _winfo_create(opal_common_ucx_wpool_t *wpool) winfo->global_inflight_ops = 0; winfo->inflight_req = UCS_OK; - WPOOL_DBG_OUT(_dbg_winfo, "winfo = %p, worker = %p\n", - (void*)winfo, (void *)winfo->worker); return winfo; release_worker: @@ -101,15 +99,11 @@ _winfo_reset(opal_common_ucx_winfo_t *winfo) winfo->endpoints = NULL; winfo->comm_size = 0; winfo->released = 0; - WPOOL_DBG_OUT(_dbg_winfo, "winfo = %p, worker = %p\n", - (void*)winfo, (void*)winfo->worker); } static void _winfo_release(opal_common_ucx_winfo_t *winfo) { - WPOOL_DBG_OUT(_dbg_winfo, "winfo = %p, worker = %p\n", - (void*)winfo, (void *)winfo->worker); OBJ_DESTRUCT(&winfo->mutex); ucp_worker_destroy(winfo->worker); free(winfo); @@ -125,7 +119,6 @@ opal_common_ucx_wpool_allocate(void) opal_common_ucx_wpool_t *ptr = calloc(1, sizeof(opal_common_ucx_wpool_t)); ptr->refcnt = 0; - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p\n", (void *)ptr); return ptr; } @@ -133,7 +126,6 @@ OPAL_DECLSPEC void opal_common_ucx_wpool_free(opal_common_ucx_wpool_t *wpool) { assert(wpool->refcnt == 0); - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p\n", (void *)wpool); free(wpool); } @@ -148,8 +140,6 @@ opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool, int rc = OPAL_SUCCESS; wpool->refcnt++; - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, refctnrt = %d\n", - (void *)wpool, wpool->refcnt); if (1 < wpool->refcnt) { return rc; @@ -213,7 +203,6 @@ opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool, pthread_key_create(&wpool->tls_key, _tlocal_cleanup); - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Done\n", (void *)wpool); return rc; err_wpool_add: @@ -233,11 +222,8 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool) { _tlocal_table_t *tls_item = NULL, *tls_next; - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Start\n", (void *)wpool); - wpool->refcnt--; if (wpool->refcnt > 0) { - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Still in use\n", (void *)wpool); return; } @@ -250,8 +236,6 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool) _tlocal_table_t) { opal_list_remove_item(&wpool->tls_list, &tls_item->super); _common_ucx_tls_cleanup(tls_item); - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Cleanup TLS = %p\n", - (void *)wpool, (void*)tls_item); } OBJ_DESTRUCT(&wpool->tls_list); @@ -265,8 +249,6 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool) OPAL_LIST_FOREACH_SAFE(item, next, &wpool->idle_workers, _winfo_list_item_t) { opal_list_remove_item(&wpool->idle_workers, &item->super); - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Cleanup idle winfo = %p\n", - (void *)wpool, (void*)item->ptr); _winfo_release(item->ptr); OBJ_RELEASE(item); } @@ -282,8 +264,6 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool) opal_list_remove_item(&wpool->active_workers, &item->super); _winfo_reset(item->ptr); _winfo_release(item->ptr); - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Cleanup active winfo = %p\n", - (void *)wpool, (void*)item->ptr); OBJ_RELEASE(item); } } @@ -291,7 +271,6 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool) OBJ_DESTRUCT(&wpool->mutex); ucp_cleanup(wpool->ucp_ctx); - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p. Done\n", (void *)wpool); return; } @@ -329,9 +308,6 @@ _wpool_list_put(opal_common_ucx_wpool_t *wpool, opal_list_t *list, { _winfo_list_item_t *item; - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, winfo = %p\n", - (void *)wpool, (void *)winfo); - item = OBJ_NEW(_winfo_list_item_t); if (NULL == item) { MCA_COMMON_UCX_ERROR("Cannot allocate memory for winfo list item"); @@ -363,8 +339,6 @@ _wpool_list_get(opal_common_ucx_wpool_t *wpool, opal_list_t *list) winfo = item->ptr; OBJ_RELEASE(item); } - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, winfo = %p\n", - (void *)wpool, (void *)winfo); return winfo; } @@ -381,9 +355,6 @@ _wpool_get_idle(opal_common_ucx_wpool_t *wpool, size_t comm_size) } } - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, winfo = %p\n", - (void *)wpool, (void *)winfo); - winfo->endpoints = calloc(comm_size, sizeof(ucp_ep_h)); winfo->inflight_ops = calloc(comm_size, sizeof(short)); winfo->comm_size = comm_size; @@ -393,8 +364,6 @@ _wpool_get_idle(opal_common_ucx_wpool_t *wpool, size_t comm_size) static int _wpool_add_active(opal_common_ucx_wpool_t *wpool, opal_common_ucx_winfo_t *winfo) { - WPOOL_DBG_OUT(_dbg_wpool, "wpool = %p, winfo = %p\n", - (void *)wpool, (void *)winfo); return _wpool_list_put(wpool, &wpool->active_workers, winfo); } @@ -411,8 +380,6 @@ opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size, opal_common_ucx_ctx_t *ctx = calloc(1, sizeof(*ctx)); int ret = OPAL_SUCCESS; - WPOOL_DBG_OUT(_dbg_ctx, "ctx_create: ctx = %p\n", (void*)ctx); - OBJ_CONSTRUCT(&ctx->mutex, opal_recursive_mutex_t); OBJ_CONSTRUCT(&ctx->tls_workers, opal_list_t); ctx->released = 0; @@ -430,8 +397,6 @@ opal_common_ucx_wpctx_create(opal_common_ucx_wpool_t *wpool, int comm_size, } (*ctx_ptr) = ctx; - WPOOL_DBG_OUT(_dbg_ctx,"wpool = %p, ctx = %p\n", - (void *)wpool, (void *)(*ctx_ptr)); return ret; error: OBJ_DESTRUCT(&ctx->mutex); @@ -446,8 +411,6 @@ opal_common_ucx_wpctx_release(opal_common_ucx_ctx_t *ctx) { int my_refcntr = -1; - WPOOL_DBG_OUT(_dbg_ctx, "ctx = %p\n", (void *)ctx); - /* Application is expected to guarantee that no operation * is performed on the context that is being released */ @@ -477,7 +440,6 @@ _common_ucx_wpctx_free(opal_common_ucx_ctx_t *ctx) free(ctx->recv_worker_displs); OBJ_DESTRUCT(&ctx->mutex); OBJ_DESTRUCT(&ctx->tls_workers); - WPOOL_DBG_OUT(_dbg_ctx, "ctx = %p\n", (void *)ctx); free(ctx); } @@ -499,8 +461,6 @@ _common_ucx_wpctx_append(opal_common_ucx_ctx_t *ctx, opal_list_append(&ctx->tls_workers, &item->super); opal_mutex_unlock(&ctx->mutex); - WPOOL_DBG_OUT(_dbg_ctx, "ctx = %p, winfo = %p\n", - (void *)ctx, (void *)winfo); return OPAL_SUCCESS; } @@ -541,7 +501,6 @@ _common_ucx_wpctx_remove(opal_common_ucx_ctx_t *ctx, * We can safely release communication context structure */ _common_ucx_wpctx_free(ctx); } - WPOOL_DBG_OUT(_dbg_ctx, "ctx = %p\n", (void *)ctx); return; } @@ -565,8 +524,6 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx, ucs_status_t status; int ret = OPAL_SUCCESS; - WPOOL_DBG_OUT(_dbg_mem, "for ctx = %p\n", (void *)ctx); - mem->released = 0; mem->refcntr = 1; /* application holding this memory handler */ mem->ctx = ctx; @@ -579,8 +536,6 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx, MCA_COMMON_UCX_VERBOSE(1, "_comm_ucx_mem_map failed: %d", ret); goto error_mem_map; } - WPOOL_DBG_OUT(_dbg_mem, "\tbase = %p, memh = %p\n", - (void *)(*mem_base), (void *)(mem->memh)); status = ucp_rkey_pack(ctx->wpool->ucp_ctx, mem->memh, &rkey_addr, &rkey_addr_len); @@ -589,12 +544,9 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx, ret = OPAL_ERROR; goto error_rkey_pack; } - WPOOL_DBG_OUT(_dbg_mem, "\trkey_addr = %p, rkey_addr_len = %d\n", - (void *)rkey_addr, (int)rkey_addr_len); ret = exchange_func(rkey_addr, rkey_addr_len, &mem->mem_addrs, &mem->mem_displs, exchange_metadata); - WPOOL_DBG_OUT(_dbg_mem, "\tcomplete exchange"); if (ret != OPAL_SUCCESS) { goto error_rkey_pack; } @@ -607,7 +559,6 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx, (*my_mem_addr) = rkey_addr; (*my_mem_addr_size) = rkey_addr_len; - WPOOL_DBG_OUT(_dbg_mem, "mem = %p. Done\n", (void *)mem); return ret; error_rkey_pack: @@ -623,8 +574,6 @@ opal_common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem) { int my_refcntr = -1; - WPOOL_DBG_OUT(_dbg_mem, "mem = %p\n", (void *)mem); - /* Mark that this memory handler has been called */ mem->released = 1; @@ -650,8 +599,6 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool, ucs_status_t status; int ret = OPAL_SUCCESS; - WPOOL_DBG_OUT(_dbg_mem, "wpool = %p\n", (void *)wpool); - memset(&mem_params, 0, sizeof(ucp_mem_map_params_t)); mem_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH | @@ -670,7 +617,6 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool, ret = OPAL_ERROR; return ret; } - WPOOL_DBG_OUT(_dbg_mem, "\tmemh = %p\n", (void *)(*memh_ptr)); mem_attrs.field_mask = UCP_MEM_ATTR_FIELD_ADDRESS | UCP_MEM_ATTR_FIELD_LENGTH; status = ucp_mem_query((*memh_ptr), &mem_attrs); @@ -679,7 +625,6 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool, ret = OPAL_ERROR; goto error; } - WPOOL_DBG_OUT(_dbg_mem, "\tmemh = %p\n", (void *)(*memh_ptr)); assert(mem_attrs.length >= size); if (mem_type != OPAL_COMMON_UCX_MEM_ALLOCATE_MAP) { @@ -688,8 +633,6 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool, (*base) = mem_attrs.address; } - WPOOL_DBG_OUT(_dbg_mem, "\twpool = %p, addr = %p size = %d memh = %p\n", - (void *)wpool, (void *)(*base), (int)size, (void *)(*memh_ptr)); return ret; error: ucp_mem_unmap(wpool->ucp_ctx, (*memh_ptr)); @@ -702,7 +645,6 @@ static void _common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem) free(mem->mem_addrs); free(mem->mem_displs); ucp_mem_unmap(mem->ctx->wpool->ucp_ctx, mem->memh); - WPOOL_DBG_OUT(_dbg_mem, "mem = %p\n", (void *)mem); free(mem); } @@ -711,8 +653,6 @@ _common_ucx_wpmem_signup(opal_common_ucx_wpmem_t *mem) { /* Increment the reference counter */ OPAL_ATOMIC_ADD_FETCH32(&mem->refcntr, 1); - - WPOOL_DBG_OUT(_dbg_mem, "mem = %p\n", (void *)mem); return OPAL_SUCCESS; } @@ -737,7 +677,6 @@ _common_ucx_mem_signout(opal_common_ucx_wpmem_t *mem) _common_ucx_wpmem_free(mem); } - WPOOL_DBG_OUT(_dbg_mem, "mem = %p\n", (void *)mem); return; } @@ -777,9 +716,6 @@ static _tlocal_table_t* _common_ucx_tls_init(opal_common_ucx_wpool_t *wpool) pthread_setspecific(wpool->tls_key, tls); - WPOOL_DBG_OUT(_dbg_tls, "tls = %p, wpool = %p\n", - (void *)tls, (void*)wpool); - return tls; } @@ -789,8 +725,6 @@ _tlocal_get_tls(opal_common_ucx_wpool_t *wpool){ if( OPAL_UNLIKELY(NULL == tls) ) { tls = _common_ucx_tls_init(wpool); } - WPOOL_DBG_OUT(_dbg_tls, "tls = %p, wpool = %p\n", - (void *)tls, (void*)wpool); return tls; } @@ -805,9 +739,6 @@ static void _tlocal_cleanup(void *arg) } wpool = tls->wpool; - WPOOL_DBG_OUT(_dbg_tls, "tls = %p, wpool = %p\n", - (void *)tls, (void*)wpool); - /* 1. Remove us from tls_list */ tls->wpool = wpool; opal_mutex_lock(&wpool->mutex); @@ -848,8 +779,6 @@ static void _common_ucx_tls_cleanup(_tlocal_table_t *tls) pthread_setspecific(tls->wpool->tls_key, NULL); - WPOOL_DBG_OUT(_dbg_tls, "tls = %p, wpool = %p\n", - (void *)tls, (void*)tls->wpool); OBJ_RELEASE(tls); return; } @@ -868,7 +797,6 @@ _tlocal_tls_ctxtbl_extend(_tlocal_table_t *tbl, size_t append) } tbl->ctx_tbl_size = newsize; - WPOOL_DBG_OUT(_dbg_tls, "new size = %d\n", (int)newsize); return OPAL_SUCCESS; } @@ -886,7 +814,6 @@ _tlocal_tls_memtbl_extend(_tlocal_table_t *tbl, size_t append) } } tbl->mem_tbl_size = newsize; - WPOOL_DBG_OUT(_dbg_tls, "new size = %d\n", (int)newsize); return OPAL_SUCCESS; } @@ -900,7 +827,6 @@ _tlocal_ctx_search(_tlocal_table_t *tls, opal_common_ucx_ctx_t *ctx) return tls->ctx_tbl[i]; } } - WPOOL_DBG_OUT(_dbg_tls, "tls = %p, ctx = %p\n", (void *)tls, (void*)ctx); return NULL; } @@ -920,10 +846,6 @@ _tlocal_ctx_record_cleanup(_tlocal_ctx_t *ctx_rec) * delayed cleanup */ _common_ucx_wpctx_remove(ctx_rec->gctx, ctx_rec->winfo); - WPOOL_DBG_OUT(_dbg_tls, "wpool = %p, winfo = %p, worker = %p\n", - (void*)ctx_rec->gctx->wpool, (void*)ctx_rec->winfo, - (void*)ctx_rec->winfo->worker); - /* Erase the record so it can be reused */ memset(ctx_rec, 0, sizeof(*ctx_rec)); @@ -987,10 +909,6 @@ _tlocal_add_ctx(_tlocal_table_t *tls, opal_common_ucx_ctx_t *ctx) return NULL; } - WPOOL_DBG_OUT(_dbg_tls || _dbg_ctx, "tls = %p, ctx_rec = %p, winfo = %p\n", - (void *)tls, (void *)&tls->ctx_tbl[free_idx], - (void *)tls->ctx_tbl[free_idx]->winfo); - /* All good - return the record */ return tls->ctx_tbl[free_idx]; } @@ -1015,8 +933,6 @@ static int _tlocal_ctx_connect(_tlocal_ctx_t *ctx_rec, int target) MCA_COMMON_UCX_VERBOSE(1, "ucp_ep_create failed: %d", status); return OPAL_ERROR; } - WPOOL_DBG_OUT(_dbg_tls || _dbg_ctx, "worker = %p ep = %p\n", - (void *)winfo->worker, (void *)winfo->endpoints[target]); opal_mutex_unlock(&winfo->mutex); return OPAL_SUCCESS; } @@ -1027,8 +943,6 @@ static inline _tlocal_mem_t * _tlocal_search_mem(_tlocal_table_t *tls, opal_common_ucx_wpmem_t *gmem) { size_t i; - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p mem = %p\n", - (void *)tls, (void*)gmem); for(i=0; imem_tbl_size; i++) { if( tls->mem_tbl[i]->gmem == gmem){ return tls->mem_tbl[i]; @@ -1041,14 +955,10 @@ static void _tlocal_mem_record_cleanup(_tlocal_mem_t *mem_rec) { size_t i; - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "record=%p, is_freed = %d\n", - (void *)mem_rec, mem_rec->gmem->released); for(i = 0; i < mem_rec->gmem->ctx->comm_size; i++) { if (mem_rec->mem->rkeys[i]) { ucp_rkey_destroy(mem_rec->mem->rkeys[i]); - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "rkey_entry = %p\n", - (void *)mem_rec->mem->rkeys[i]); } } free(mem_rec->mem->rkeys); @@ -1057,8 +967,6 @@ _tlocal_mem_record_cleanup(_tlocal_mem_t *mem_rec) * This may result in context release as we are using * delayed cleanup */ _common_ucx_mem_signout(mem_rec->gmem); - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "gmem = %p mem_rec = %p\n", - (void *)mem_rec->gmem, (void *)mem_rec); /* Release fast-path pointers */ if (NULL != mem_rec->mem_tls_ptr) { @@ -1103,7 +1011,6 @@ static _tlocal_mem_t *_tlocal_add_mem(_tlocal_table_t *tls, //TODO: error out return NULL; } - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p\n", (void *)tls); } tls->mem_tbl[free_idx]->gmem = mem; @@ -1114,8 +1021,6 @@ static _tlocal_mem_t *_tlocal_add_mem(_tlocal_table_t *tls, // TODO: act accordingly - cleanup return NULL; } - WPOOL_DBG_OUT("tls = %p, ctx = %p\n", - (void *)tls, (void*)mem->ctx); tls->mem_tbl[free_idx]->ctx_rec = ctx_rec; OPAL_ATOMIC_ADD_FETCH32(&ctx_rec->refcnt, 1); @@ -1142,9 +1047,6 @@ static _tlocal_mem_t *_tlocal_add_mem(_tlocal_table_t *tls, // TODO: error handling return NULL; } - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, - "mem = %p, mem_tbl_entry = %p\n", - (void *)mem, (void *)tls->mem_tbl[free_idx]); return tls->mem_tbl[free_idx]; } @@ -1163,8 +1065,7 @@ _tlocal_mem_create_rkey(_tlocal_mem_t *mem_rec, ucp_ep_h ep, int target) MCA_COMMON_UCX_VERBOSE(1, "ucp_ep_rkey_unpack failed: %d", status); return OPAL_ERROR; } - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "mem_rec = %p ep = %p target = %d\n", - (void *)mem_rec, (void *)ep, target); + return OPAL_SUCCESS; } @@ -1181,24 +1082,17 @@ opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target) int rc = OPAL_SUCCESS; tls = _tlocal_get_tls(mem->ctx->wpool); - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p\n",(void*)tls); /* Obtain the worker structure */ ctx_rec = _tlocal_ctx_search(tls, mem->ctx); - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "ctx_id = %p, ctx_rec=%p\n", - (void*)mem->ctx, (void *)ctx_rec); if (OPAL_UNLIKELY(NULL == ctx_rec)) { ctx_rec = _tlocal_add_ctx(tls, mem->ctx); if (NULL == ctx_rec) { return OPAL_ERR_OUT_OF_RESOURCE; } - WPOOL_DBG_OUT("_tlocal_fetch(after _tlocal_add_ctx): tls = %p ctx = %p\n", - (void *)tls, (void *)mem->ctx); } winfo = ctx_rec->winfo; - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "winfo = %p ctx=%p\n", - (void *)winfo, (void *)mem->ctx); /* Obtain the endpoint */ if (OPAL_UNLIKELY(NULL == winfo->endpoints[target])) { @@ -1206,26 +1100,18 @@ opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target) if (rc != OPAL_SUCCESS) { return rc; } - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "ctx_rec = %p target = %d\n", - (void *)ctx_rec, target); } ep = winfo->endpoints[target]; - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "ep = %p\n", (void *)ep); /* Obtain the memory region info */ mem_rec = _tlocal_search_mem(tls, mem); - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p mem_rec = %p mem_id = %p\n", - (void *)tls, (void *)mem_rec, (void*)mem); if (OPAL_UNLIKELY(mem_rec == NULL)) { mem_rec = _tlocal_add_mem(tls, mem); - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "tls = %p mem = %p\n", - (void *)tls, (void *)mem); if (NULL == mem_rec) { return OPAL_ERR_OUT_OF_RESOURCE; } } mem_info = mem_rec->mem; - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "mem_info = %p\n", (void *)mem_info); /* Obtain the rkey */ if (OPAL_UNLIKELY(NULL == mem_info->rkeys[target])) { @@ -1234,7 +1120,6 @@ opal_common_ucx_tlocal_fetch_spath(opal_common_ucx_wpmem_t *mem, int target) if (rc) { return rc; } - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "creating rkey ...\n"); } return OPAL_SUCCESS; @@ -1293,9 +1178,6 @@ opal_common_ucx_wpmem_flush(opal_common_ucx_wpmem_t *mem, opal_common_ucx_ctx_t *ctx = mem->ctx; int rc = OPAL_SUCCESS; - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, "mem = %p, target = %d\n", - (void *)mem, target); - opal_mutex_lock(&ctx->mutex); OPAL_LIST_FOREACH(item, &ctx->tls_workers, _ctx_record_list_item_t) { @@ -1323,11 +1205,6 @@ opal_common_ucx_wpmem_flush(opal_common_ucx_wpmem_t *mem, rc); rc = OPAL_ERROR; } - WPOOL_DBG_OUT(_dbg_tls || _dbg_mem, - "target = %d, ep = %p worker = %p\n", - (int)target, - (void *)item->ptr->endpoints[target], - (void *)item->ptr->worker); } opal_mutex_unlock(&ctx->mutex); diff --git a/opal/mca/common/ucx/common_ucx_wpool.h b/opal/mca/common/ucx/common_ucx_wpool.h index 3c103dfdbb..9accc924d1 100644 --- a/opal/mca/common/ucx/common_ucx_wpool.h +++ b/opal/mca/common/ucx/common_ucx_wpool.h @@ -127,68 +127,6 @@ typedef int (*opal_common_ucx_exchange_func_t)(void *my_info, size_t my_info_len char **recv_info, int **disps, void *metadata); -/* For developer use only */ -//#define OPAL_COMMON_UCX_WPOOL_DBG -#ifdef OPAL_COMMON_UCX_WPOOL_DBG -extern __thread FILE *tls_pf; -extern __thread int initialized; - -#include -#include -#include -#include - -static int _dbg_winfo = 0; -static int _dbg_wpool = 0; -static int _dbg_ctx = 0; -static int _dbg_mem = 0; -static int _dbg_tls = 0; - -static inline void opal_common_ucx_wpool_dbg_init(void) -{ - if( !initialized ) { - int tid = syscall(__NR_gettid); - char hname[128]; - gethostname(hname, 127); - char fname[128]; - - sprintf(fname, "%s.%d.log", hname, tid); - tls_pf = fopen(fname, "w"); - initialized = 1; - // Create issusion that they are used to avoid compiler warnings - (void)_dbg_ctx; - (void)_dbg_mem; - (void)_dbg_tls; - (void)_dbg_winfo; - (void)_dbg_wpool; - } -} - - -#define WPOOL_DBG_OUT(level, ...) \ -{ \ - struct timeval start_; \ - time_t nowtime_; \ - struct tm *nowtm_; \ - char tmbuf_[64]; \ - gettimeofday(&start_, NULL); \ - nowtime_ = start_.tv_sec; \ - nowtm_ = localtime(&nowtime_); \ - strftime(tmbuf_, sizeof(tmbuf_), \ - "%H:%M:%S", nowtm_); \ - opal_common_ucx_wpool_dbg_init(); \ - if (level) { \ - fprintf(tls_pf, "[%s.%06ld] %s:", \ - tmbuf_, start_.tv_usec, \ - __func__); \ - } \ - fprintf(tls_pf, __VA_ARGS__); \ -} - -#else -#define WPOOL_DBG_OUT(...) -#endif - /* Manage Worker Pool (wpool) */ OPAL_DECLSPEC opal_common_ucx_wpool_t * opal_common_ucx_wpool_allocate(void); @@ -374,8 +312,6 @@ opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem, opal_common_ucx_op_t MCA_COMMON_UCX_VERBOSE(1, "tlocal_fetch failed: %d", rc); return rc; } - WPOOL_DBG_OUT(_dbg_mem, "mem = %p, ep = %p, rkey = %p, winfo = %p\n", - (void *)mem, (void *)ep, (void *)rkey, (void *)winfo); /* Perform the operation */ opal_mutex_lock(&winfo->mutex); @@ -393,9 +329,6 @@ opal_common_ucx_wpmem_putget(opal_common_ucx_wpmem_t *mem, opal_common_ucx_op_t if (OPAL_UNLIKELY(status != UCS_OK && status != UCS_INPROGRESS)) { MCA_COMMON_UCX_ERROR("%s failed: %d", called_func, status); rc = OPAL_ERROR; - } else { - WPOOL_DBG_OUT(_dbg_mem,"ep = %p, rkey = %p\n", - (void *)ep, (void *)rkey); } rc = _periodical_flush_nb(mem, winfo, target); @@ -426,8 +359,6 @@ opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare, MCA_COMMON_UCX_ERROR("opal_common_ucx_tlocal_fetch failed: %d", rc); return rc; } - WPOOL_DBG_OUT("mem = %p, ep = %p, rkey = %p, winfo = %p\n", - (void *)mem, (void *)ep, (void *)rkey, (void *)winfo); /* Perform the operation */ opal_mutex_lock(&winfo->mutex); @@ -438,9 +369,6 @@ opal_common_ucx_wpmem_cmpswp(opal_common_ucx_wpmem_t *mem, uint64_t compare, if (OPAL_UNLIKELY(status != UCS_OK)) { MCA_COMMON_UCX_ERROR("opal_common_ucx_atomic_cswap failed: %d", status); rc = OPAL_ERROR; - } else { - WPOOL_DBG_OUT(_dbg_mem, "ep = %p, rkey = %p\n", - (void *)ep, (void *)rkey); } rc = _periodical_flush_nb(mem, winfo, target); @@ -470,8 +398,6 @@ opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem, ucp_atomic_post_op_t op MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc); return rc; } - WPOOL_DBG_OUT(_dbg_mem, "mem = %p, ep = %p, rkey = %p, winfo = %p\n", - (void *)mem, (void *)ep, (void *)rkey, (void *)winfo); /* Perform the operation */ opal_mutex_lock(&winfo->mutex); @@ -480,9 +406,6 @@ opal_common_ucx_wpmem_post(opal_common_ucx_wpmem_t *mem, ucp_atomic_post_op_t op if (OPAL_UNLIKELY(status != UCS_OK)) { MCA_COMMON_UCX_ERROR("ucp_atomic_post failed: %d", status); rc = OPAL_ERROR; - } else { - WPOOL_DBG_OUT(_dbg_mem, "ep = %p, rkey = %p\n", - (void *)ep, (void *)rkey); } rc = _periodical_flush_nb(mem, winfo, target); @@ -512,8 +435,6 @@ opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem, MCA_COMMON_UCX_ERROR("tlocal_fetch failed: %d", rc); return rc; } - WPOOL_DBG_OUT(_dbg_mem, "mem = %p, ep = %p, rkey = %p, winfo = %p\n", - (void *)mem, (void *)ep, (void *)rkey, (void *)winfo); /* Perform the operation */ opal_mutex_lock(&winfo->mutex); @@ -524,9 +445,6 @@ opal_common_ucx_wpmem_fetch(opal_common_ucx_wpmem_t *mem, if (OPAL_UNLIKELY(status != UCS_OK)) { MCA_COMMON_UCX_ERROR("ucp_atomic_cswap64 failed: %d", status); rc = OPAL_ERROR; - } else { - WPOOL_DBG_OUT(_dbg_mem, "ep = %p, rkey = %p\n", - (void *)ep, (void *)rkey); } rc = _periodical_flush_nb(mem, winfo, target);