From 3933d0a8f0b933e2c79a7cf7b0b955e806205d2f Mon Sep 17 00:00:00 2001 From: Brian Barrett Date: Wed, 27 Jun 2012 15:58:17 +0000 Subject: [PATCH] Ibarrier works! :) This commit was SVN r26680. --- ompi/mca/coll/libnbc/coll_libnbc.h | 70 +++++--- ompi/mca/coll/libnbc/coll_libnbc_component.c | 160 +++++++++++++++---- ompi/mca/coll/libnbc/nbc.c | 144 +++++------------ ompi/mca/coll/libnbc/nbc.h | 8 - ompi/mca/coll/libnbc/nbc_internal.h | 4 +- ompi/request/request_dbg.h | 1 + 6 files changed, 218 insertions(+), 169 deletions(-) diff --git a/ompi/mca/coll/libnbc/coll_libnbc.h b/ompi/mca/coll/libnbc/coll_libnbc.h index a1ff6c3f23..6967396bea 100644 --- a/ompi/mca/coll/libnbc/coll_libnbc.h +++ b/ompi/mca/coll/libnbc/coll_libnbc.h @@ -25,20 +25,6 @@ BEGIN_C_DECLS -/* Globally exported variables */ -OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_2_0_0_t mca_coll_libnbc_component; - -int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_0_0_t *module); - - -struct ompi_coll_libnbc_module_t { - mca_coll_base_module_t super; -}; -typedef struct ompi_coll_libnbc_module_t ompi_coll_libnbc_module_t; -OBJ_CLASS_DECLARATION(ompi_coll_libnbc_module_t); - - /* Function return codes */ #define NBC_OK 0 /* everything went fine */ #define NBC_SUCCESS 0 /* everything went fine (MPI compliant :) */ @@ -54,14 +40,20 @@ OBJ_CLASS_DECLARATION(ompi_coll_libnbc_module_t); /* number of implemented collective functions */ #define NBC_NUM_COLL 19 -/* a schedule is basically a pointer to some memory location where the - * schedule array resides */ -typedef void* NBC_Schedule; +struct ompi_coll_libnbc_component_t { + mca_coll_base_component_2_0_0_t super; + ompi_free_list_t requests; + opal_list_t active_requests; + uint32_t active_comms; +}; +typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t; -/* used to hang off a communicator */ -typedef struct { - MPI_Comm mycomm; /* save the shadow communicator here */ - int tag; +/* Globally exported variables */ +OMPI_MODULE_DECLSPEC extern ompi_coll_libnbc_component_t mca_coll_libnbc_component; + +struct ompi_coll_libnbc_module_t { + mca_coll_base_module_t super; + int tag; #ifdef NBC_CACHE_SCHEDULE void *NBC_Dict[NBC_NUM_COLL]; /* this should point to a struct hb_tree, but since this is a @@ -70,12 +62,19 @@ typedef struct { it ...*/ int NBC_Dict_size[NBC_NUM_COLL]; #endif -} NBC_Comminfo; +}; +typedef struct ompi_coll_libnbc_module_t ompi_coll_libnbc_module_t; +OBJ_CLASS_DECLARATION(ompi_coll_libnbc_module_t); + +typedef ompi_coll_libnbc_module_t NBC_Comminfo; + +/* a schedule is basically a pointer to some memory location where the + * schedule array resides */ +typedef void* NBC_Schedule; struct ompi_coll_libnbc_request_t { ompi_request_t super; MPI_Comm comm; - MPI_Comm mycomm; long row_offset; int tag; volatile int req_count; @@ -92,6 +91,31 @@ OBJ_CLASS_DECLARATION(ompi_coll_libnbc_request_t); typedef ompi_coll_libnbc_request_t NBC_Handle; + +#define OMPI_COLL_LIBNBC_REQUEST_ALLOC(comm, req, rc) \ + do { \ + ompi_free_list_item_t *item; \ + OMPI_FREE_LIST_WAIT(&mca_coll_libnbc_component.requests, \ + item, rc); \ + req = (ompi_coll_libnbc_request_t*) item; \ + OMPI_REQUEST_INIT(&req->super, false); \ + req->super.req_mpi_object.comm = comm; \ + req->super.req_complete = false; \ + req->super.req_state = OMPI_REQUEST_ACTIVE; \ + } while (0) + +#define OMPI_COLL_LIBNBC_REQUEST_RETURN(req) \ + do { \ + OMPI_REQUEST_FINI(&request->super); \ + OMPI_FREE_LIST_RETURN(&mca_coll_libnbc_component.requests, \ + (ompi_free_list_item_t*) req); \ + } while (0) + +int NBC_Init_comm(MPI_Comm comm, ompi_coll_libnbc_module_t *module); +int NBC_Progress(NBC_Handle *handle); +int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_0_0_t *module); + END_C_DECLS #endif /* MCA_COLL_LIBNBC_EXPORT_H */ diff --git a/ompi/mca/coll/libnbc/coll_libnbc_component.c b/ompi/mca/coll/libnbc/coll_libnbc_component.c index 9e3f6d1fd0..5d6faa1665 100644 --- a/ompi/mca/coll/libnbc/coll_libnbc_component.c +++ b/ompi/mca/coll/libnbc/coll_libnbc_component.c @@ -35,55 +35,86 @@ const char *mca_coll_libnbc_component_version_string = static int libnbc_priority = 10; +static int libnbc_open(void); +static int libnbc_close(void); static int libnbc_register(void); static int libnbc_init_query(bool, bool); static mca_coll_base_module_t *libnbc_comm_query(struct ompi_communicator_t *, int *); static int libnbc_module_enable(mca_coll_base_module_t *, struct ompi_communicator_t *); - +static int libnbc_progress(void); /* * Instantiate the public struct with all of our public information * and pointers to our public functions in it */ -const mca_coll_base_component_2_0_0_t mca_coll_libnbc_component = { - - /* First, the mca_component_t struct containing meta information - * about the component itself */ - +ompi_coll_libnbc_component_t mca_coll_libnbc_component = { { - MCA_COLL_BASE_VERSION_2_0_0, + /* First, the mca_component_t struct containing meta information + * about the component itself */ + { + MCA_COLL_BASE_VERSION_2_0_0, - /* Component name and version */ - "libnbc", - OMPI_MAJOR_VERSION, - OMPI_MINOR_VERSION, - OMPI_RELEASE_VERSION, + /* Component name and version */ + "libnbc", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION, - /* Component open and close functions */ - NULL, - NULL, - NULL, - libnbc_register - }, - { - /* The component is checkpoint ready */ - MCA_BASE_METADATA_PARAM_CHECKPOINT - }, + /* Component open and close functions */ + libnbc_open, + libnbc_close, + NULL, + libnbc_register + }, + { + /* The component is checkpoint ready */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, - /* Initialization / querying functions */ - - libnbc_init_query, - libnbc_comm_query + /* Initialization / querying functions */ + libnbc_init_query, + libnbc_comm_query + } }; +static int +libnbc_open(void) +{ + int ret; + + OBJ_CONSTRUCT(&mca_coll_libnbc_component.requests, ompi_free_list_t); + ret = ompi_free_list_init(&mca_coll_libnbc_component.requests, + sizeof(ompi_coll_libnbc_request_t), + OBJ_CLASS(ompi_coll_libnbc_request_t), + 0, + -1, + 8, + NULL); + if (OMPI_SUCCESS != ret) return ret; + + OBJ_CONSTRUCT(&mca_coll_libnbc_component.active_requests, opal_list_t); + mca_coll_libnbc_component.active_comms = 0; + + return OMPI_SUCCESS; +} + +static int +libnbc_close(void) +{ + OBJ_DESTRUCT(&mca_coll_libnbc_component.requests); + + return OMPI_SUCCESS; +} + + static int libnbc_register(void) { /* Use a low priority, but allow other components to be lower */ - mca_base_param_reg_int(&mca_coll_libnbc_component.collm_version, + mca_base_param_reg_int(&mca_coll_libnbc_component.super.collm_version, "priority", "Priority of the libnbc coll component", false, false, libnbc_priority, @@ -130,6 +161,11 @@ libnbc_comm_query(struct ompi_communicator_t *comm, module->super.ft_event = NULL; + if (OMPI_SUCCESS != NBC_Init_comm(comm, module)) { + OBJ_RELEASE(module); + return NULL; + } + return &(module->super); } @@ -142,18 +178,47 @@ libnbc_module_enable(mca_coll_base_module_t *module, struct ompi_communicator_t *comm) { /* All done */ + if (0 == mca_coll_libnbc_component.active_comms++) { + opal_progress_register(libnbc_progress); + } return OMPI_SUCCESS; } +static int +libnbc_progress(void) +{ + opal_list_item_t *item; + + for (item = opal_list_get_first(&mca_coll_libnbc_component.active_requests) ; + item != opal_list_get_end(&mca_coll_libnbc_component.active_requests) ; + item = opal_list_get_next(item)) { + ompi_coll_libnbc_request_t* request = (ompi_coll_libnbc_request_t*) item; + if (NBC_OK == NBC_Progress(request)) { + request->super.req_status.MPI_ERROR = OMPI_SUCCESS; + /* done, remove */ + item = opal_list_remove_item(&mca_coll_libnbc_component.active_requests, + &request->super.super.super); + } + item = opal_list_get_next(item); + } + + return 0; +} + + static void libnbc_module_construct(ompi_coll_libnbc_module_t *module) { } + static void libnbc_module_destruct(ompi_coll_libnbc_module_t *module) { + if (0 == --mca_coll_libnbc_component.active_comms) { + opal_progress_unregister(libnbc_progress); + } } @@ -161,3 +226,44 @@ OBJ_CLASS_INSTANCE(ompi_coll_libnbc_module_t, mca_coll_base_module_t, libnbc_module_construct, libnbc_module_destruct); + + +static int +request_cancel(struct ompi_request_t *request, int complete) +{ + return MPI_ERR_REQUEST; +} + + +static int +request_free(struct ompi_request_t **ompi_req) +{ + ompi_coll_libnbc_request_t *request = + (ompi_coll_libnbc_request_t*) *ompi_req; + + if (true != request->super.req_complete) { + return MPI_ERR_REQUEST; + } + + OMPI_COLL_LIBNBC_REQUEST_RETURN(request); + + *ompi_req = MPI_REQUEST_NULL; + + return OMPI_SUCCESS; +} + + +static void +request_construct(ompi_coll_libnbc_request_t *request) +{ + request->super.req_type = OMPI_REQUEST_COLL; + request->super.req_status._cancelled = 0; + request->super.req_free = request_free; + request->super.req_cancel = request_cancel; +} + + +OBJ_CLASS_INSTANCE(ompi_coll_libnbc_request_t, + ompi_request_t, + request_construct, + NULL); diff --git a/ompi/mca/coll/libnbc/nbc.c b/ompi/mca/coll/libnbc/nbc.c index fd42bc43c6..98d04d9309 100644 --- a/ompi/mca/coll/libnbc/nbc.c +++ b/ompi/mca/coll/libnbc/nbc.c @@ -25,33 +25,6 @@ void NBC_Print_times(double div) { } #endif -/* is NBC globally initialized */ -static char GNBC_Initialized=0; - -/* the keyval (global) */ -static int gkeyval=MPI_KEYVAL_INVALID; - -static int NBC_Key_copy(MPI_Comm oldcomm, int keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag) { - /* delete the attribute in the new comm - it will be created at the - * first usage */ - *flag = 0; - - return MPI_SUCCESS; -} - -static int NBC_Key_delete(MPI_Comm comm, int keyval, void *attribute_val, void *extra_state) { - NBC_Comminfo *comminfo; - - if(keyval == gkeyval) { - comminfo=(NBC_Comminfo*)attribute_val; - free((void*)comminfo); - } else { - printf("Got wrong keyval!(%i)\n", keyval); - } - - return MPI_SUCCESS; -} - /* allocates a new schedule array */ int NBC_Sched_create(NBC_Schedule* schedule) { @@ -299,6 +272,10 @@ static inline int NBC_Free(NBC_Handle* handle) { handle->tmpbuf = NULL; } + OPAL_THREAD_LOCK(&ompi_request_lock); + ompi_request_complete(&handle->super, true); + OPAL_THREAD_UNLOCK(&ompi_request_lock); + return NBC_OK; } @@ -416,10 +393,10 @@ static inline int NBC_Start_round(NBC_Handle *handle) { #ifdef HAVE_OMPI handle->req_array = (MPI_Request*)realloc((void*)handle->req_array, (handle->req_count)*sizeof(MPI_Request)); NBC_CHECK_NULL(handle->req_array); - /*res = MCA_PML_CALL(isend_init(buf1, sendargs->count, sendargs->datatype, sendargs->dest, handle->tag, MCA_PML_BASE_SEND_STANDARD, handle->mycomm, handle->req_array+handle->req_count-1)); - printf("MPI_Isend(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, sendargs->count, (unsigned long)sendargs->datatype, sendargs->dest, handle->tag, (unsigned long)handle->mycomm, res);*/ - res = MPI_Isend(buf1, sendargs->count, sendargs->datatype, sendargs->dest, handle->tag, handle->mycomm, handle->req_array+handle->req_count-1); - if(OMPI_SUCCESS != res) { printf("Error in MPI_Isend(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, sendargs->count, (unsigned long)sendargs->datatype, sendargs->dest, handle->tag, (unsigned long)handle->mycomm, res); ret=res; goto error; } + /*res = MCA_PML_CALL(isend_init(buf1, sendargs->count, sendargs->datatype, sendargs->dest, handle->tag, MCA_PML_BASE_SEND_STANDARD, handle->comm, handle->req_array+handle->req_count-1)); + printf("MPI_Isend(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, sendargs->count, (unsigned long)sendargs->datatype, sendargs->dest, handle->tag, (unsigned long)handle->comm, res);*/ + res = MPI_Isend(buf1, sendargs->count, sendargs->datatype, sendargs->dest, handle->tag, handle->comm, handle->req_array+handle->req_count-1); + if(OMPI_SUCCESS != res) { printf("Error in MPI_Isend(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, sendargs->count, (unsigned long)sendargs->datatype, sendargs->dest, handle->tag, (unsigned long)handle->comm, res); ret=res; goto error; } #endif #ifdef NBC_TIMING Isend_time += MPI_Wtime(); @@ -444,10 +421,10 @@ static inline int NBC_Start_round(NBC_Handle *handle) { #ifdef HAVE_OMPI handle->req_array = (MPI_Request*)realloc((void*)handle->req_array, (handle->req_count)*sizeof(MPI_Request)); NBC_CHECK_NULL(handle->req_array); - /*res = MCA_PML_CALL(irecv(buf1, recvargs->count, recvargs->datatype, recvargs->source, handle->tag, handle->mycomm, handle->req_array+handle->req_count-1)); - printf("MPI_Irecv(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, recvargs->count, (unsigned long)recvargs->datatype, recvargs->source, handle->tag, (unsigned long)handle->mycomm, res); */ - res = MPI_Irecv(buf1, recvargs->count, recvargs->datatype, recvargs->source, handle->tag, handle->mycomm, handle->req_array+handle->req_count-1); - if(OMPI_SUCCESS != res) { printf("Error in MPI_Irecv(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, recvargs->count, (unsigned long)recvargs->datatype, recvargs->source, handle->tag, (unsigned long)handle->mycomm, res); ret=res; goto error; } + /*res = MCA_PML_CALL(irecv(buf1, recvargs->count, recvargs->datatype, recvargs->source, handle->tag, handle->comm, handle->req_array+handle->req_count-1)); + printf("MPI_Irecv(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, recvargs->count, (unsigned long)recvargs->datatype, recvargs->source, handle->tag, (unsigned long)handle->comm, res); */ + res = MPI_Irecv(buf1, recvargs->count, recvargs->datatype, recvargs->source, handle->tag, handle->comm, handle->req_array+handle->req_count-1); + if(OMPI_SUCCESS != res) { printf("Error in MPI_Irecv(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, recvargs->count, (unsigned long)recvargs->datatype, recvargs->source, handle->tag, (unsigned long)handle->comm, res); ret=res; goto error; } #endif #ifdef NBC_TIMING Irecv_time += MPI_Wtime(); @@ -488,7 +465,7 @@ static inline int NBC_Start_round(NBC_Handle *handle) { buf2=(char*)handle->tmpbuf+(long)copyargs->tgt; else buf2=copyargs->tgt; - res = NBC_Copy(buf1, copyargs->srccount, copyargs->srctype, buf2, copyargs->tgtcount, copyargs->tgttype, handle->mycomm); + res = NBC_Copy(buf1, copyargs->srccount, copyargs->srctype, buf2, copyargs->tgtcount, copyargs->tgttype, handle->comm); if(res != NBC_OK) { printf("NBC_Copy() failed (code: %i)\n", res); ret=res; goto error; } break; case UNPACK: @@ -505,7 +482,7 @@ static inline int NBC_Start_round(NBC_Handle *handle) { buf2=(char*)handle->tmpbuf+(long)unpackargs->outbuf; else buf2=unpackargs->outbuf; - res = NBC_Unpack(buf1, unpackargs->count, unpackargs->datatype, buf2, handle->mycomm); + res = NBC_Unpack(buf1, unpackargs->count, unpackargs->datatype, buf2, handle->comm); if(res != NBC_OK) { printf("NBC_Unpack() failed (code: %i)\n", res); ret=res; goto error; } break; default: @@ -531,18 +508,14 @@ error: return ret; } -static inline int NBC_Initialize(void) { - GNBC_Initialized = 1; - - return NBC_OK; -} - -int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t **request, ompi_coll_libnbc_module_t *module) +int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t **request, ompi_coll_libnbc_module_t *comminfo) { - int res, flag; - NBC_Comminfo *comminfo; + int res; + ompi_coll_libnbc_request_t *handle; - NBC_Handle *handle = NULL; + OMPI_COLL_LIBNBC_REQUEST_ALLOC(comm, handle, res); + if (OMPI_SUCCESS != res) return res; + *request = handle; handle->tmpbuf = NULL; handle->req_count = 0; @@ -553,27 +526,11 @@ int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t handle->row_offset = sizeof(int); /******************** Do the tag and shadow comm administration ... ***************/ - - /* otherwise we have to do the normal attribute stuff :-( */ - /* keyval is not initialized yet, we have to init it */ - if(MPI_KEYVAL_INVALID == gkeyval) { - res = MPI_Keyval_create(NBC_Key_copy, NBC_Key_delete, &(gkeyval), NULL); - if((MPI_SUCCESS != res)) { printf("Error in MPI_Keyval_create() (%i)\n", res); return res; } - } - res = MPI_Attr_get(comm, gkeyval, &comminfo, &flag); - if((MPI_SUCCESS != res)) { printf("Error in MPI_Attr_get() (%i)\n", res); return res; } - - if (flag) { - /* we found it */ - comminfo->tag++; - } else { - /* we have to create a new one */ - comminfo = NBC_Init_comm(comm); - if(comminfo == NULL) { printf("Error in NBC_Init_comm() %i\n", res); return NBC_OOR; } - } + /* we found it */ + comminfo->tag++; handle->tag=comminfo->tag; - handle->mycomm=comminfo->mycomm; + handle->comm=comm; /*printf("got comminfo: %lu tag: %i\n", comminfo, comminfo->tag);*/ /* reset counter ... */ @@ -591,35 +548,25 @@ int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t return NBC_OK; } -NBC_Comminfo* NBC_Init_comm(MPI_Comm comm) { - int res; - NBC_Comminfo *comminfo; - - comminfo = (NBC_Comminfo*)malloc(sizeof(NBC_Comminfo)); - if(comminfo == NULL) { printf("Error in malloc()\n"); return NULL; } - +int NBC_Init_comm(MPI_Comm comm, NBC_Comminfo *comminfo) { /* set tag to 1 */ comminfo->tag=1; - /* dup and save shadow communicator */ - res = MPI_Comm_dup(comm, &(comminfo->mycomm)); - if((MPI_SUCCESS != res)) { printf("Error in MPI_Comm_dup() (%i)\n", res); return NULL; } - NBC_DEBUG(1, "created a shadow communicator for %lu ... %lu\n", (unsigned long)comm, (unsigned long)comminfo->mycomm); #ifdef NBC_CACHE_SCHEDULE #if 0 /* initialize the NBC_ALLTOALL SchedCache tree */ comminfo->NBC_Dict[NBC_ALLTOALL] = hb_tree_new((dict_cmp_func)NBC_Alltoall_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_ALLTOALL] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_ALLTOALL] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_ALLTOALL]); comminfo->NBC_Dict_size[NBC_ALLTOALL] = 0; /* initialize the NBC_ALLGATHER SchedCache tree */ comminfo->NBC_Dict[NBC_ALLGATHER] = hb_tree_new((dict_cmp_func)NBC_Allgather_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_ALLGATHER] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_ALLGATHER] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_ALLGATHER]); comminfo->NBC_Dict_size[NBC_ALLGATHER] = 0; /* initialize the NBC_ALLREDUCE SchedCache tree */ comminfo->NBC_Dict[NBC_ALLREDUCE] = hb_tree_new((dict_cmp_func)NBC_Allreduce_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_ALLREDUCE] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_ALLREDUCE] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_ALLREDUCE]); comminfo->NBC_Dict_size[NBC_ALLREDUCE] = 0; #endif @@ -629,47 +576,43 @@ NBC_Comminfo* NBC_Init_comm(MPI_Comm comm) { #if 0 /* initialize the NBC_BCAST SchedCache tree */ comminfo->NBC_Dict[NBC_BCAST] = hb_tree_new((dict_cmp_func)NBC_Bcast_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_BCAST] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_BCAST] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_BCAST]); comminfo->NBC_Dict_size[NBC_BCAST] = 0; /* initialize the NBC_GATHER SchedCache tree */ comminfo->NBC_Dict[NBC_GATHER] = hb_tree_new((dict_cmp_func)NBC_Gather_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_GATHER] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_GATHER] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_GATHER]); comminfo->NBC_Dict_size[NBC_GATHER] = 0; /* initialize the NBC_REDUCE SchedCache tree */ comminfo->NBC_Dict[NBC_REDUCE] = hb_tree_new((dict_cmp_func)NBC_Reduce_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_REDUCE] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_REDUCE] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_REDUCE]); comminfo->NBC_Dict_size[NBC_REDUCE] = 0; /* initialize the NBC_SCAN SchedCache tree */ comminfo->NBC_Dict[NBC_SCAN] = hb_tree_new((dict_cmp_func)NBC_Scan_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_SCAN] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_SCAN] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_SCAN]); comminfo->NBC_Dict_size[NBC_SCAN] = 0; /* initialize the NBC_SCATTER SchedCache tree */ comminfo->NBC_Dict[NBC_SCATTER] = hb_tree_new((dict_cmp_func)NBC_Scatter_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_SCATTER] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_SCATTER] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_SCATTER]); comminfo->NBC_Dict_size[NBC_SCATTER] = 0; /* initialize the NBC_ICART_SHIFT_XCHG SchedCache tree */ comminfo->NBC_Dict[NBC_CART_SHIFT_XCHG] = hb_tree_new((dict_cmp_func)NBC_Icart_shift_xchg_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_CART_SHIFT_XCHG] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_CART_SHIFT_XCHG] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_CART_SHIFT_XCHG]); comminfo->NBC_Dict_size[NBC_CART_SHIFT_XCHG] = 0; /* initialize the NBC_INEIGHBOR_XCHG SchedCache tree */ comminfo->NBC_Dict[NBC_NEIGHBOR_XCHG] = hb_tree_new((dict_cmp_func)NBC_Ineighbor_xchg_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete); - if(comminfo->NBC_Dict[NBC_NEIGHBOR_XCHG] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; } + if(comminfo->NBC_Dict[NBC_NEIGHBOR_XCHG] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; } NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_NEIGHBOR_XCHG]); comminfo->NBC_Dict_size[NBC_NEIGHBOR_XCHG] = 0; #endif #endif - /* put the new attribute to the comm */ - res = MPI_Attr_put(comm, gkeyval, comminfo); - if((MPI_SUCCESS != res)) { printf("Error in MPI_Attr_put() (%i)\n", res); return NULL; } - - return comminfo; + return OMPI_SUCCESS; } int NBC_Start(NBC_Handle *handle, NBC_Schedule *schedule) { @@ -681,26 +624,11 @@ int NBC_Start(NBC_Handle *handle, NBC_Schedule *schedule) { res = NBC_Start_round(handle); if((NBC_OK != res)) { printf("Error in NBC_Start_round() (%i)\n", res); return res; } - return NBC_OK; -} - -int NBC_Wait(NBC_Handle *handle) { - /* poll */ - while(NBC_OK != NBC_Progress(handle)); - - NBC_DEBUG(3, "finished request with tag %i\n", handle->tag); - - return NBC_OK; -} - -int NBC_Test(NBC_Handle *handle, int *flag, MPI_Status *status) { - int ret = NBC_Progress(handle); - *flag = ret; + opal_list_append(&mca_coll_libnbc_component.active_requests, &(handle->super.super.super)); return NBC_OK; } - #ifdef NBC_CACHE_SCHEDULE void NBC_SchedCache_args_delete_key_dummy(void *k) { /* do nothing because the key and the data element are identical :-) diff --git a/ompi/mca/coll/libnbc/nbc.h b/ompi/mca/coll/libnbc/nbc.h index d53ac6ddc2..f13333edb4 100644 --- a/ompi/mca/coll/libnbc/nbc.h +++ b/ompi/mca/coll/libnbc/nbc.h @@ -36,14 +36,6 @@ int NBC_Ireduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, int NBC_Iallreduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, NBC_Handle* handle); int NBC_Iscan(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, NBC_Handle* handle); int NBC_Ireduce_scatter(void* sendbuf, void* recvbuf, int *recvcounts, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, NBC_Handle* handle); -int NBC_Icart_shift_xchg(void *sbuf, int scount, MPI_Datatype stype, void *rbuf, int rcount, MPI_Datatype rtype, int direction, int disp, MPI_Comm comm, NBC_Handle* handle); -int NBC_Ineighbor_xchg(void *sbuf, int scount, MPI_Datatype stype, void *rbuf, int rcount, MPI_Datatype rtype, MPI_Comm comm, NBC_Handle* handle); -int NBC_Ineighbor_xchgv(void *sbuf, int *scounts, int *sdispls, MPI_Datatype stype, void *rbuf, int *rcounts, int *rdispls, MPI_Datatype rtype, MPI_Comm comm, NBC_Handle* handle); - -int NBC_Comm_neighbors(MPI_Comm comm, int maxindegree, int sources[], int sourceweights[], int maxoutdegree, int destinations[], int destweights[]); -int NBC_Comm_neighbors_count(MPI_Comm comm, int *indegree, int *outdegree, int *weighted); -int NBC_Wait(NBC_Handle *handle); -int NBC_Test(NBC_Handle *handle, int *flag, MPI_Status *status); /* TODO: some hacks */ int NBC_Operation(void *buf3, void *buf1, void *buf2, MPI_Op op, MPI_Datatype type, int count); diff --git a/ompi/mca/coll/libnbc/nbc_internal.h b/ompi/mca/coll/libnbc/nbc_internal.h index e525a3f98a..d65950dc68 100644 --- a/ompi/mca/coll/libnbc/nbc_internal.h +++ b/ompi/mca/coll/libnbc/nbc_internal.h @@ -28,7 +28,7 @@ * still in the cache and second, the tmpbuf used by the schedule must * be attached to the handle that uses this schedule !!!! * I.E., THIS IS EXPERIMENTAL AND MIGHT NOT WORK */ -#define NBC_CACHE_SCHEDULE +/* #define NBC_CACHE_SCHEDULE */ #define NBC_SCHED_DICT_UPPER 1024 /* max. number of dict entries */ #define NBC_SCHED_DICT_LOWER 512 /* nuber of dict entries after wipe, if SCHED_DICT_UPPER is reached */ @@ -288,12 +288,10 @@ void NBC_SchedCache_args_delete_key_dummy(void *k); #endif -int NBC_Progress(NBC_Handle *handle); int NBC_Start(NBC_Handle *handle, NBC_Schedule *schedule); int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t **request, ompi_coll_libnbc_module_t *module); static inline int NBC_Type_intrinsic(MPI_Datatype type); static inline int NBC_Copy(void *src, int srccount, MPI_Datatype srctype, void *tgt, int tgtcount, MPI_Datatype tgttype, MPI_Comm comm); -NBC_Comminfo* NBC_Init_comm(MPI_Comm comm); int NBC_Create_fortran_handle(int *fhandle, NBC_Handle **handle); /* some macros */ diff --git a/ompi/request/request_dbg.h b/ompi/request/request_dbg.h index 580585bc15..e4421f4e0b 100644 --- a/ompi/request/request_dbg.h +++ b/ompi/request/request_dbg.h @@ -24,6 +24,7 @@ typedef enum { OMPI_REQUEST_IO, /**< MPI-2 IO request */ OMPI_REQUEST_GEN, /**< MPI-2 generalized request */ OMPI_REQUEST_WIN, /**< MPI-2 one-sided request */ + OMPI_REQUEST_COLL, /**< MPI-3 non-blocking collectives request */ OMPI_REQUEST_NULL, /**< NULL request */ OMPI_REQUEST_NOOP, /**< A request that does nothing (e.g., to PROC_NULL) */ OMPI_REQUEST_MAX /**< Maximum request type */