diff --git a/ompi/mca/btl/openib/Makefile.am b/ompi/mca/btl/openib/Makefile.am index 4c012c5725..3a6c3b7cf6 100644 --- a/ompi/mca/btl/openib/Makefile.am +++ b/ompi/mca/btl/openib/Makefile.am @@ -5,16 +5,16 @@ # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. # $COPYRIGHT$ -# +# # Additional copyrights may follow -# +# # $HEADER$ # @@ -116,10 +116,10 @@ endif mcacomponentdir = $(pkglibdir) mcacomponent_LTLIBRARIES = $(component) mca_btl_openib_la_SOURCES = $(component_sources) -mca_btl_openib_la_LDFLAGS = -module -avoid-version $(btl_openib_LDFLAGS) +mca_btl_openib_la_LDFLAGS = -module -avoid-version $(btl_openib_LDFLAGS) mca_btl_openib_la_LIBADD = $(btl_openib_LIBS) noinst_LTLIBRARIES = $(lib) libmca_btl_openib_la_SOURCES = $(lib_sources) -libmca_btl_openib_la_LDFLAGS= -module -avoid-version $(btl_openib_LDFLAGS) +libmca_btl_openib_la_LDFLAGS= -module -avoid-version $(btl_openib_LDFLAGS) libmca_btl_openib_la_LIBADD = $(btl_openib_LIBS) diff --git a/ompi/mca/btl/openib/btl_openib.c b/ompi/mca/btl/openib/btl_openib.c index 5dbfc1edc3..297e31e07a 100644 --- a/ompi/mca/btl/openib/btl_openib.c +++ b/ompi/mca/btl/openib/btl_openib.c @@ -413,8 +413,8 @@ static int mca_btl_openib_size_queues(struct mca_btl_openib_module_t* openib_btl goto out; } - if (0 == openib_btl->num_peers && - (mca_btl_openib_component.num_srq_qps > 0 || + if (0 == openib_btl->num_peers && + (mca_btl_openib_component.num_srq_qps > 0 || mca_btl_openib_component.num_xrc_qps > 0)) { rc = create_srq(openib_btl); } @@ -426,7 +426,7 @@ out: mca_btl_openib_transport_type_t mca_btl_openib_get_transport_type(mca_btl_openib_module_t* openib_btl) { -/* If we have a driver with RDMAoE supporting as the device struct contains the same type (IB) for +/* If we have a driver with RDMAoE supporting as the device struct contains the same type (IB) for IBV_LINK_LAYER_INFINIBAND and IBV_LINK_LAYER_ETHERNET link layers and the single way to detect this fact is to check their link_layer fields in a port_attr struct. If our driver doesn't support this feature => the checking of transport type in device struct will be enough. @@ -455,7 +455,7 @@ mca_btl_openib_transport_type_t mca_btl_openib_get_transport_type(mca_btl_openib case IBV_TRANSPORT_IWARP: return MCA_BTL_OPENIB_TRANSPORT_IWARP; - case IBV_TRANSPORT_UNKNOWN: + case IBV_TRANSPORT_UNKNOWN: default: return MCA_BTL_OPENIB_TRANSPORT_UNKNOWN; } @@ -464,7 +464,7 @@ mca_btl_openib_transport_type_t mca_btl_openib_get_transport_type(mca_btl_openib #endif } -static int mca_btl_openib_tune_endpoint(mca_btl_openib_module_t* openib_btl, +static int mca_btl_openib_tune_endpoint(mca_btl_openib_module_t* openib_btl, mca_btl_base_endpoint_t* endpoint) { int ret = OMPI_SUCCESS; @@ -485,7 +485,7 @@ static int mca_btl_openib_tune_endpoint(mca_btl_openib_module_t* openib_btl, endpoint->rem_info.rem_vendor_id, endpoint->rem_info.rem_vendor_part_id, mca_btl_openib_transport_name_strings[endpoint->rem_info.rem_transport_type]); - + return OMPI_ERROR; } @@ -503,7 +503,7 @@ static int mca_btl_openib_tune_endpoint(mca_btl_openib_module_t* openib_btl, } if(openib_btl->device->mtu < endpoint->rem_info.rem_mtu) { - endpoint->rem_info.rem_mtu = openib_btl->device->mtu; + endpoint->rem_info.rem_mtu = openib_btl->device->mtu; } endpoint->use_eager_rdma = openib_btl->device->use_eager_rdma & @@ -520,11 +520,11 @@ static int mca_btl_openib_tune_endpoint(mca_btl_openib_module_t* openib_btl, case BTL_OPENIB_RQ_SOURCE_MAX: break; - /* If the queues configuration was set from command line + /* If the queues configuration was set from command line (with --mca btl_openib_receive_queues parameter) => both sides have a same configuration */ /* In this case the local queues configuration was gotten from INI file => - not possible that remote side got its queues configuration from command line => + not possible that remote side got its queues configuration from command line => (by prio) the configuration was set from INI file or (if not configure) by default queues configuration */ case BTL_OPENIB_RQ_SOURCE_DEVICE_INI: @@ -552,7 +552,7 @@ static int mca_btl_openib_tune_endpoint(mca_btl_openib_module_t* openib_btl, } break; - /* If the local queues configuration was set + /* If the local queues configuration was set by default queues => check all possible cases for remote side and compare */ case BTL_OPENIB_RQ_SOURCE_DEFAULT: if(NULL != values.receive_queues) { @@ -628,7 +628,7 @@ int mca_btl_openib_add_procs( opal_output(-1, "add procs: adding proc %d", i); - /* OOB, XOOB, RDMACM, IBCM does not support SELF comunication, so + /* OOB, XOOB, RDMACM, IBCM does not support SELF comunication, so * mark the prco as unreachable by openib btl */ if (OPAL_EQUAL == orte_util_compare_name_fields (ORTE_NS_CMP_ALL, ORTE_PROC_MY_NAME, &ompi_proc->proc_name)) { @@ -705,7 +705,7 @@ int mca_btl_openib_add_procs( on the peer has a matching CPC. */ assert(btl_rank <= ib_proc->proc_port_count); assert(remote_matching_port != -1); - if (OMPI_SUCCESS != + if (OMPI_SUCCESS != ompi_btl_openib_connect_base_find_match(openib_btl, &(ib_proc->proc_ports[remote_matching_port]), &local_cpc, @@ -751,8 +751,8 @@ int mca_btl_openib_add_procs( } } #endif - mca_btl_openib_endpoint_init(openib_btl, endpoint, - local_cpc, + mca_btl_openib_endpoint_init(openib_btl, endpoint, + local_cpc, &(ib_proc->proc_ports[remote_matching_port]), remote_cpc_data); @@ -1139,7 +1139,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src( BTL_VERBOSE(("frag->sg_entry.lkey = %" PRIu32 " .addr = %" PRIx64 " frag->segment.seg_key.key32[0] = %" PRIu32, - frag->sg_entry.lkey, + frag->sg_entry.lkey, frag->sg_entry.addr, frag->sg_entry.lkey)); @@ -1260,7 +1260,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst( BTL_VERBOSE(("frag->sg_entry.lkey = %" PRIu32 " .addr = %" PRIx64 " " "frag->segment.seg_key.key32[0] = %" PRIu32, - frag->sg_entry.lkey, + frag->sg_entry.lkey, frag->sg_entry.addr, openib_reg->mr->rkey)); @@ -1317,7 +1317,7 @@ static int mca_btl_openib_finalize_resources(struct mca_btl_base_module_t* btl) opal_hash_table_t *srq_addr_table = &mca_btl_openib_component.srq_manager.srq_addr_table; - opal_mutex_lock(lock); + opal_mutex_lock(lock); if (OPAL_SUCCESS != opal_hash_table_remove_value_ptr(srq_addr_table, &openib_btl->qps[qp].u.srq_qp.srq, @@ -1353,7 +1353,7 @@ static int mca_btl_openib_finalize_resources(struct mca_btl_base_module_t* btl) } if (NULL != openib_btl->qps) { - free(openib_btl->qps); + free(openib_btl->qps); } return rc; @@ -1399,7 +1399,7 @@ int mca_btl_openib_finalize(struct mca_btl_base_module_t* btl) /* * Send immediate - Minimum function calls minimum checks, send the data ASAP. - * If BTL can't to send the messages imidiate, it creates messages descriptor + * If BTL can't to send the messages imidiate, it creates messages descriptor * returns it to PML. */ int mca_btl_openib_sendi( struct mca_btl_base_module_t* btl, @@ -1411,14 +1411,14 @@ int mca_btl_openib_sendi( struct mca_btl_base_module_t* btl, uint8_t order, uint32_t flags, mca_btl_base_tag_t tag, - mca_btl_base_descriptor_t** descriptor) + mca_btl_base_descriptor_t** descriptor) { mca_btl_openib_module_t *obtl = (mca_btl_openib_module_t*)btl; size_t size = payload_size + header_size; size_t eager_limit; - int rc, - qp = frag_size_to_order(obtl, size), - prio = !(flags & MCA_BTL_DES_FLAGS_PRIORITY), + int rc, + qp = frag_size_to_order(obtl, size), + prio = !(flags & MCA_BTL_DES_FLAGS_PRIORITY), ib_rc; int32_t cm_return; bool do_rdma = false; diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index 9d21b059e4..5da4996947 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -142,7 +142,7 @@ typedef enum { /* The structer for manage all BTL SRQs */ typedef struct mca_btl_openib_srq_manager_t { opal_mutex_t lock; - /* The keys of this hash table are addresses of + /* The keys of this hash table are addresses of SRQs structures, and the elements are BTL modules pointers that associated with these SRQs */ opal_hash_table_t srq_addr_table; @@ -405,7 +405,7 @@ struct mca_btl_openib_module_srq_qp_t { /** We post additional WQEs only if a number of WQEs (in specific SRQ) is less of this value. The value increased together with rd_curr_num. The value is unique for every SRQ. */ int32_t rd_low_local; - /** The flag points if we want to get the + /** The flag points if we want to get the IBV_EVENT_SRQ_LIMIT_REACHED events for dynamically resizing SRQ */ bool srq_limit_event_flag; /**< In difference of the "--mca enable_srq_resize" parameter that says, if we want(or no) @@ -580,7 +580,7 @@ extern int mca_btl_openib_sendi( struct mca_btl_base_module_t* btl, uint32_t flags, mca_btl_base_tag_t tag, mca_btl_base_descriptor_t** descriptor -); +); /** * PML->BTL Initiate a put of the specified size. diff --git a/ompi/mca/btl/openib/btl_openib_async.c b/ompi/mca/btl/openib/btl_openib_async.c index 839f2aa8ff..76e7b6df27 100644 --- a/ompi/mca/btl/openib/btl_openib_async.c +++ b/ompi/mca/btl/openib/btl_openib_async.c @@ -141,7 +141,7 @@ static int btl_openib_async_poll_init(struct mca_btl_openib_async_poll *devices_ } /* Send command completion to main thread */ -static int send_command_comp(int in) +static int send_command_comp(int in) { if (write(mca_btl_openib_component.async_comp_pipe[1], &in, sizeof(int)) < 0) { BTL_ERROR(("Write failed [%d]",errno)); @@ -227,7 +227,7 @@ static int btl_openib_async_commandh(struct mca_btl_openib_async_poll *devices_p return OMPI_SUCCESS; } -/* The main idea of resizing SRQ algorithm - +/* The main idea of resizing SRQ algorithm - We create a SRQ with size = rd_num, but for efficient usage of resources the number of WQEs that we post = rd_curr_num < rd_num and this value is increased (by needs) in IBV_EVENT_SRQ_LIMIT_REACHED event handler (i.e. in this function), @@ -327,7 +327,7 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po BTL_ERROR(("Alternative path migration event reported")); if (APM_ENABLED) { BTL_ERROR(("Trying to find additional path...")); - if (!xrc_event) + if (!xrc_event) mca_btl_openib_load_apm(event.element.qp, qp2endpoint(event.element.qp, device)); #if HAVE_XRC @@ -472,7 +472,7 @@ void* btl_openib_async_thread(void * async) return PTHREAD_CANCELED; } -int btl_openib_async_command_done(int exp) +int btl_openib_async_command_done(int exp) { int comp; if (read(mca_btl_openib_component.async_comp_pipe[0], &comp, diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index b9994dfe59..84c8914ca5 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -284,27 +284,27 @@ static int btl_openib_modex_send(void) * c. a uint8_t indicating the length of the blob to follow * d. a blob that is only meaningful to that CPC */ - msg_size = + msg_size = /* uint8_t for number of modules in the message */ 1 + /* For each module: */ - mca_btl_openib_component.ib_num_btls * + mca_btl_openib_component.ib_num_btls * ( /* Common module data */ - modex_message_size + + modex_message_size + /* uint8_t for how many CPCs follow */ 1 ); /* For each module, add in the size of the per-CPC data */ for (i = 0; i < mca_btl_openib_component.ib_num_btls; i++) { - for (j = 0; + for (j = 0; j < mca_btl_openib_component.openib_btls[i]->num_cpcs; ++j) { - msg_size += + msg_size += /* uint8_t for the index of the CPC */ 1 + /* uint8_t for the CPC's priority */ - 1 + + 1 + /* uint8_t for the blob length */ 1 + /* blob length */ @@ -337,15 +337,15 @@ static int btl_openib_modex_send(void) (mca_btl_openib_component.openib_btls[i]->port_info).transport_type = mca_btl_openib_get_transport_type(mca_btl_openib_component.openib_btls[i]); - memcpy(offset, - &(mca_btl_openib_component.openib_btls[i]->port_info), + memcpy(offset, + &(mca_btl_openib_component.openib_btls[i]->port_info), size); opal_output(-1, "modex packed btl port modex message: 0x%" PRIx64 ", %d, %d (size: %d)", mca_btl_openib_component.openib_btls[i]->port_info.subnet_id, mca_btl_openib_component.openib_btls[i]->port_info.mtu, mca_btl_openib_component.openib_btls[i]->port_info.lid, (int) size); - + #if !defined(WORDS_BIGENDIAN) && OPAL_ENABLE_HETEROGENEOUS_SUPPORT MCA_BTL_OPENIB_MODEX_MSG_HTON(*(mca_btl_openib_modex_message_t *)offset); #endif @@ -354,20 +354,20 @@ static int btl_openib_modex_send(void) i, (int) (offset -message)); /* Pack the number of CPCs that follow */ - pack8(&offset, + pack8(&offset, mca_btl_openib_component.openib_btls[i]->num_cpcs); opal_output(-1, "modex packed btl %d: to pack %d cpcs (packed: %d, offset now %d)", i, mca_btl_openib_component.openib_btls[i]->num_cpcs, *((uint8_t*) (offset - 1)), (int) (offset-message)); /* Pack each CPC */ - for (j = 0; + for (j = 0; j < mca_btl_openib_component.openib_btls[i]->num_cpcs; ++j) { uint8_t u8; cpc = mca_btl_openib_component.openib_btls[i]->cpcs[j]; - opal_output(-1, "modex packed btl %d: packing cpc %s", + opal_output(-1, "modex packed btl %d: packing cpc %s", i, cpc->data.cbm_component->cbc_name); /* Pack the CPC index */ u8 = ompi_btl_openib_connect_base_get_cpc_index(cpc->data.cbm_component); @@ -434,7 +434,7 @@ static void btl_openib_control(mca_btl_base_module_t* btl, case MCA_BTL_OPENIB_CONTROL_RDMA: rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)ctl_hdr; - BTL_VERBOSE(("prior to NTOH received rkey %" PRIu32 + BTL_VERBOSE(("prior to NTOH received rkey %" PRIu32 ", rdma_start.lval %" PRIx64 ", pval %p, ival %" PRIu32, rdma_hdr->rkey, rdma_hdr->rdma_start.lval, @@ -446,7 +446,7 @@ static void btl_openib_control(mca_btl_base_module_t* btl, BTL_OPENIB_EAGER_RDMA_CONTROL_HEADER_NTOH(*rdma_hdr); } - BTL_VERBOSE(("received rkey %" PRIu32 + BTL_VERBOSE(("received rkey %" PRIu32 ", rdma_start.lval %" PRIx64 ", pval %p," " ival %" PRIu32, rdma_hdr->rkey, rdma_hdr->rdma_start.lval, @@ -633,7 +633,7 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_device_t *device, BTL_VERBOSE(("my iWARP subnet_id is %016" PRIx64, subnet_id)); } else { memset(&gid, 0, sizeof(gid)); - if (0 != ibv_query_gid(device->ib_dev_context, port_num, + if (0 != ibv_query_gid(device->ib_dev_context, port_num, mca_btl_openib_component.gid_index, &gid)) { BTL_ERROR(("ibv_query_gid failed (%s:%d, %d)\n", ibv_get_device_name(device->ib_dev), port_num, @@ -643,7 +643,7 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_device_t *device, #ifdef OMPI_HAVE_RDMAOE if (IBV_LINK_LAYER_ETHERNET == ib_port_attr->link_layer) { - subnet_id = mca_btl_openib_get_ip_subnet_id(device->ib_dev, + subnet_id = mca_btl_openib_get_ip_subnet_id(device->ib_dev, port_num); } else { subnet_id = ntoh64(gid.global.subnet_prefix); @@ -652,11 +652,11 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_device_t *device, subnet_id = ntoh64(gid.global.subnet_prefix); #endif - BTL_VERBOSE(("my IB subnet_id for HCA %s port %d is %016" PRIx64, + BTL_VERBOSE(("my IB subnet_id for HCA %s port %d is %016" PRIx64, ibv_get_device_name(device->ib_dev), port_num, subnet_id)); } #else - if (0 != ibv_query_gid(device->ib_dev_context, port_num, + if (0 != ibv_query_gid(device->ib_dev_context, port_num, mca_btl_openib_component.gid_index, &gid)) { BTL_ERROR(("ibv_query_gid failed (%s:%d, %d)\n", ibv_get_device_name(device->ib_dev), port_num, @@ -664,7 +664,7 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_device_t *device, return OMPI_ERR_NOT_FOUND; } subnet_id = ntoh64(gid.global.subnet_prefix); - BTL_VERBOSE(("my IB-only subnet_id for HCA %s port %d is %016" PRIx64, + BTL_VERBOSE(("my IB-only subnet_id for HCA %s port %d is %016" PRIx64, ibv_get_device_name(device->ib_dev), port_num, subnet_id)); #endif @@ -868,9 +868,9 @@ static void device_construct(mca_btl_openib_device_t *device) #endif device->qps = NULL; #if OPAL_HAVE_THREADS - mca_btl_openib_component.async_pipe[0] = + mca_btl_openib_component.async_pipe[0] = mca_btl_openib_component.async_pipe[1] = -1; - mca_btl_openib_component.async_comp_pipe[0] = + mca_btl_openib_component.async_comp_pipe[0] = mca_btl_openib_component.async_comp_pipe[1] = -1; #endif OBJ_CONSTRUCT(&device->device_lock, opal_mutex_t); @@ -910,7 +910,7 @@ static void device_destruct(mca_btl_openib_device_t *device) /* wait for ok from thread */ if (OMPI_SUCCESS != btl_openib_async_command_done(device_to_remove)){ goto device_error; - } + } } #endif @@ -1006,10 +1006,10 @@ static int prepare_device_for_use(mca_btl_openib_device_t *device) return OMPI_ERROR; } /* wait for ok from thread */ - if (OMPI_SUCCESS != + if (OMPI_SUCCESS != btl_openib_async_command_done(device->ib_dev_context->async_fd)) { return OMPI_ERROR; - } + } } #if OMPI_ENABLE_PROGRESS_THREADS == 1 /* Prepare data for thread, but not starting it */ @@ -1338,7 +1338,7 @@ static int setup_qps(void) if (0 == opal_argv_count(queues)) { orte_show_help("help-mpi-btl-openib.txt", "no qps in receive_queues", true, - orte_process_info.nodename, + orte_process_info.nodename, mca_btl_openib_component.receive_queues); ret = OMPI_ERROR; goto error; @@ -1357,7 +1357,7 @@ static int setup_qps(void) num_xrc_qps++; #else orte_show_help("help-mpi-btl-openib.txt", "No XRC support", true, - orte_process_info.nodename, + orte_process_info.nodename, mca_btl_openib_component.receive_queues); ret = OMPI_ERR_NOT_AVAILABLE; goto error; @@ -1365,7 +1365,7 @@ static int setup_qps(void) } else { orte_show_help("help-mpi-btl-openib.txt", "invalid qp type in receive_queues", true, - orte_process_info.nodename, + orte_process_info.nodename, mca_btl_openib_component.receive_queues, queues[qp]); ret = OMPI_ERR_BAD_PARAM; @@ -1377,7 +1377,7 @@ static int setup_qps(void) and SRQ */ if (num_xrc_qps > 0 && (num_pp_qps > 0 || num_srq_qps > 0)) { orte_show_help("help-mpi-btl-openib.txt", "XRC with PP or SRQ", true, - orte_process_info.nodename, + orte_process_info.nodename, mca_btl_openib_component.receive_queues); ret = OMPI_ERR_BAD_PARAM; goto error; @@ -1385,8 +1385,8 @@ static int setup_qps(void) /* Current XRC implementation can't used with btls_per_lid > 1 */ if (num_xrc_qps > 0 && mca_btl_openib_component.btls_per_lid > 1) { - orte_show_help("help-mpi-btl-openib.txt", "XRC with BTLs per LID", - true, orte_process_info.nodename, + orte_show_help("help-mpi-btl-openib.txt", "XRC with BTLs per LID", + true, orte_process_info.nodename, mca_btl_openib_component.receive_queues, num_xrc_qps); ret = OMPI_ERR_BAD_PARAM; goto error; @@ -1676,13 +1676,13 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) ports and QPs on this device */ need_search = false; if(-2 != mca_btl_openib_component.ib_max_inline_data) { - /* User has explicitly set btl_openib_max_inline_data MCA parameter + /* User has explicitly set btl_openib_max_inline_data MCA parameter Per setup in _mca.c, we know that the MCA param value is guaranteed to be >= -1 */ if (-1 == mca_btl_openib_component.ib_max_inline_data) { need_search = true; } else { - device->max_inline_data = (uint32_t) + device->max_inline_data = (uint32_t) mca_btl_openib_component.ib_max_inline_data; } } else if (values.max_inline_data_set) { @@ -1691,20 +1691,20 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) } else if (values.max_inline_data >= 0) { device->max_inline_data = (uint32_t) values.max_inline_data; } else { - if(default_values.max_inline_data_set && + if(default_values.max_inline_data_set && default_values.max_inline_data >= -1) { BTL_ERROR(("Invalid max_inline_data value specified " - "in INI file (%d); using default value (%d)", - values.max_inline_data, + "in INI file (%d); using default value (%d)", + values.max_inline_data, default_values.max_inline_data)); - device->max_inline_data = (uint32_t) + device->max_inline_data = (uint32_t) default_values.max_inline_data; } else { BTL_ERROR(("Invalid max_inline_data value specified " "in INI file (%d)", values.max_inline_data)); ret = OMPI_ERR_BAD_PARAM; goto error; - } + } } } /* Horrible. :-( Per the thread starting here: @@ -1729,7 +1729,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) orte_show_help("help-mpi-btl-openib.txt", "init-fail-create-q", true, orte_process_info.nodename, __FILE__, __LINE__, "ibv_create_cq", - strerror(errno), errno, + strerror(errno), errno, ibv_get_device_name(device->ib_dev)); ret = OMPI_ERR_NOT_AVAILABLE; goto error; @@ -1749,7 +1749,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) that's good enough */ init_attr.cap.max_inline_data = max_inline_data = 1 << 20; while (max_inline_data > 0) { - qp = ibv_create_qp(device->ib_pd, &init_attr); + qp = ibv_create_qp(device->ib_pd, &init_attr); if (NULL != qp) { break; } @@ -1779,7 +1779,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) /* Eager RDMA is not currently supported with progress threads */ if (device->use_eager_rdma && OMPI_ENABLE_PROGRESS_THREADS) { device->use_eager_rdma = 0; - orte_show_help("help-mpi-btl-openib.txt", + orte_show_help("help-mpi-btl-openib.txt", "eager RDMA and progress threads", true); } @@ -1896,7 +1896,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) modules in this process. This is an unfortunate artifact of the openib BTL startup sequence (see below for more details). The first device will choose the receive_queues - value from: (in priority order): + value from: (in priority order): 1. If the btl_openib_receive_queues MCA param was specified, use that. @@ -1991,7 +1991,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) shares one of the problems cited in case 8, below. So we need to fail this scenario; print an error and abort. - + Case 8: one INI value, different than default - MCA parameter: not specified - default receive_queues: value A @@ -2036,7 +2036,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) Server 2: HCA B: no receive_queues in INI file HCA C: receive_queues specified in INI file - + A will therefore use the default receive_queues value. B and C will use C's INI receive_queues. But note that modex [currently] only sends around @@ -2078,7 +2078,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) - device 1: receive_queues value B in INI file - device 2: receive_queues value B in INI file --> per case 8, fail with a show_help message. - + Case 10: two devices with different INI values - MCA parameter: not specified - default receive_queues: value A @@ -2121,7 +2121,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) in the INI, then it must agree with component.receive_queues. */ if (NULL != values.receive_queues) { - if (0 != strcmp(values.receive_queues, + if (0 != strcmp(values.receive_queues, mca_btl_openib_component.receive_queues)) { orte_show_help("help-mpi-btl-openib.txt", "locally conflicting receive_queues", true, @@ -2187,7 +2187,7 @@ error: if (OMPI_SUCCESS != ret) { orte_show_help("help-mpi-btl-openib.txt", - "error in device init", true, + "error in device init", true, orte_process_info.nodename, ibv_get_device_name(device->ib_dev)); } @@ -2453,7 +2453,7 @@ btl_openib_component_init(int *num_btl_modules, support, so the following test is [currently] good enough... */ value = opal_mem_hooks_support_level(); #if !OPAL_HAVE_THREADS - if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) == + if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) == ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) { orte_show_help("help-mpi-btl-openib.txt", "ptmalloc2 with no threads", true, @@ -2468,7 +2468,7 @@ btl_openib_component_init(int *num_btl_modules, We have a memory manager if we have both FREE and MUNMAP support */ - if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) == + if ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) == ((OPAL_MEMORY_FREE_SUPPORT | OPAL_MEMORY_MUNMAP_SUPPORT) & value)) { ret = 0; index = mca_base_param_find("mpi", NULL, "leave_pinned"); @@ -2481,7 +2481,7 @@ btl_openib_component_init(int *num_btl_modules, index = mca_base_param_find("mpi", NULL, "leave_pinned_pipeline"); if (index >= 0) { if (OPAL_SUCCESS == mca_base_param_lookup_int(index, &value) && - OPAL_SUCCESS == mca_base_param_lookup_source(index, &source, + OPAL_SUCCESS == mca_base_param_lookup_source(index, &source, NULL)) { if (0 == value && MCA_BASE_PARAM_SOURCE_DEFAULT == source) { ++ret; @@ -2498,10 +2498,10 @@ btl_openib_component_init(int *num_btl_modules, if (index >= 0) { if (OPAL_SUCCESS == mca_base_param_lookup_source(index, &source, NULL)) { - if (-1 == mca_btl_openib_component.ib_max_inline_data && + if (-1 == mca_btl_openib_component.ib_max_inline_data && MCA_BASE_PARAM_SOURCE_DEFAULT == source) { /* If the user has not explicitly set this MCA parameter - use max_inline_data value specified in the + use max_inline_data value specified in the device-specific parameters INI file */ mca_btl_openib_component.ib_max_inline_data = -2; } @@ -2644,7 +2644,7 @@ btl_openib_component_init(int *num_btl_modules, mca_btl_openib_component.async_thread = 0; #endif distance = dev_sorted[0].distance; - for (found = false, i = 0; + for (found = false, i = 0; i < num_devs && (-1 == mca_btl_openib_component.ib_max_btls || mca_btl_openib_component.ib_num_btls < mca_btl_openib_component.ib_max_btls); i++) { @@ -2740,7 +2740,7 @@ btl_openib_component_init(int *num_btl_modules, } #endif - /* For XRC: + /* For XRC: * from this point we know if MCA_BTL_XRC_ENABLED it true or false */ /* Init XRC IB Addr hash table */ @@ -2807,11 +2807,11 @@ btl_openib_component_init(int *num_btl_modules, base device that doesn't have device->qps setup on it yet (remember that some modules may share the same device, so when going through to loop, we may hit a device that was already setup earlier in - the loop). - + the loop). + We may to call for prepare_device_for_use() only after adding the btl - to mca_btl_openib_component.openib_btls, since the prepare_device_for_use - adds device to async thread that require access to + to mca_btl_openib_component.openib_btls, since the prepare_device_for_use + adds device to async thread that require access to mca_btl_openib_component.openib_btls. */ @@ -2829,7 +2829,7 @@ btl_openib_component_init(int *num_btl_modules, ret = prepare_device_for_use(device); if (OMPI_SUCCESS != ret) { orte_show_help("help-mpi-btl-openib.txt", - "error in device init", true, + "error in device init", true, orte_process_info.nodename, ibv_get_device_name(device->ib_dev)); goto no_btls; @@ -2856,12 +2856,12 @@ btl_openib_component_init(int *num_btl_modules, opal_argv_free(mca_btl_openib_component.if_exclude_list); mca_btl_openib_component.if_exclude_list = NULL; } - + /* setup the fork warning message as we are sensitive * to memory corruption issues when fork is called */ ompi_warn_fork(); - + return btls; no_btls: @@ -2923,7 +2923,7 @@ static int progress_no_credits_pending_frags(mca_btl_base_endpoint_t *ep) retransmit for us). */ for (len = opal_list_get_size(&ep->qps[qp].no_credits_pending_frags[pri]); - len > 0 && + len > 0 && (ep->eager_rdma_remote.tokens > 0 || ep->qps[qp].u.pp_qp.sd_credits > 0 || !BTL_OPENIB_QP_TYPE_PP(qp)); --len) { @@ -3095,7 +3095,7 @@ static int btl_openib_handle_incoming(mca_btl_openib_module_t *openib_btl, if (rcredits + credits > 0) { int rc; - if (OMPI_SUCCESS != + if (OMPI_SUCCESS != (rc = progress_no_credits_pending_frags(ep))) { return rc; } @@ -3363,7 +3363,7 @@ error: * SQ and RQ WRs will automatically be flushed. */ #if defined(HAVE_STRUCT_IBV_DEVICE_TRANSPORT_TYPE) - if (IBV_WC_WR_FLUSH_ERR == wc->status && + if (IBV_WC_WR_FLUSH_ERR == wc->status && IBV_TRANSPORT_IWARP == device->ib_dev->transport_type) { return; } @@ -3373,47 +3373,47 @@ error: BTL_PEER_ERROR(remote_proc, ("error polling %s with status %s " "status number %d for wr_id %" PRIx64 " opcode %d vendor error %d qp_idx %d", cq_name[cq], btl_openib_component_status_to_string(wc->status), - wc->status, wc->wr_id, + wc->status, wc->wr_id, wc->opcode, wc->vendor_err, qp)); orte_notifier.log_peer(ORTE_NOTIFIER_CRIT, ORTE_ERR_COMM_FAILURE, remote_proc ? &remote_proc->proc_name : NULL, "\n\tIB polling %s with status %s " "status number %d for wr_id %llu opcode %d vendor error %d qp_idx %d", cq_name[cq], btl_openib_component_status_to_string(wc->status), - wc->status, wc->wr_id, + wc->status, wc->wr_id, wc->opcode, wc->vendor_err, qp); } if (IBV_WC_RNR_RETRY_EXC_ERR == wc->status || IBV_WC_RETRY_EXC_ERR == wc->status) { - char *peer_hostname = + char *peer_hostname = (NULL != endpoint->endpoint_proc->proc_ompi->proc_hostname) ? - endpoint->endpoint_proc->proc_ompi->proc_hostname : + endpoint->endpoint_proc->proc_ompi->proc_hostname : ""; - const char *device_name = + const char *device_name = ibv_get_device_name(endpoint->qps[qp].qp->lcl_qp->context->device); if (IBV_WC_RNR_RETRY_EXC_ERR == wc->status) { orte_show_help("help-mpi-btl-openib.txt", - BTL_OPENIB_QP_TYPE_PP(qp) ? - "pp rnr retry exceeded" : + BTL_OPENIB_QP_TYPE_PP(qp) ? + "pp rnr retry exceeded" : "srq rnr retry exceeded", true, orte_process_info.nodename, device_name, peer_hostname); orte_notifier.show_help(ORTE_NOTIFIER_CRIT, ORTE_ERR_COMM_FAILURE, "help-mpi-btl-openib.txt", - BTL_OPENIB_QP_TYPE_PP(qp) ? - "pp rnr retry exceeded" : + BTL_OPENIB_QP_TYPE_PP(qp) ? + "pp rnr retry exceeded" : "srq rnr retry exceeded", orte_process_info.nodename, device_name, peer_hostname); } else if (IBV_WC_RETRY_EXC_ERR == wc->status) { - orte_show_help("help-mpi-btl-openib.txt", + orte_show_help("help-mpi-btl-openib.txt", "pp retry exceeded", true, orte_process_info.nodename, device_name, peer_hostname); orte_notifier.show_help(ORTE_NOTIFIER_CRIT, ORTE_ERR_COMM_FAILURE, - "help-mpi-btl-openib.txt", + "help-mpi-btl-openib.txt", "pp retry exceeded", orte_process_info.nodename, device_name, peer_hostname); diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.c b/ompi/mca/btl/openib/btl_openib_endpoint.c index 93001c72eb..243b884533 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.c +++ b/ompi/mca/btl/openib/btl_openib_endpoint.c @@ -310,7 +310,7 @@ void mca_btl_openib_endpoint_init(mca_btl_openib_module_t *btl, ep->rem_info.rem_lid = remote_proc_info->pm_port_info.lid; ep->rem_info.rem_subnet_id = remote_proc_info->pm_port_info.subnet_id; ep->rem_info.rem_mtu = remote_proc_info->pm_port_info.mtu; - opal_output(-1, "Got remote LID, subnet, MTU: %d, 0x%" PRIx64 ", %d", + opal_output(-1, "Got remote LID, subnet, MTU: %d, 0x%" PRIx64 ", %d", ep->rem_info.rem_lid, ep->rem_info.rem_subnet_id, ep->rem_info.rem_mtu); @@ -508,7 +508,7 @@ static void cts_sent(mca_btl_base_module_t* btl, /* * Send CTS control fragment */ -void mca_btl_openib_endpoint_send_cts(mca_btl_openib_endpoint_t *endpoint) +void mca_btl_openib_endpoint_send_cts(mca_btl_openib_endpoint_t *endpoint) { mca_btl_openib_send_control_frag_t *sc_frag; mca_btl_base_descriptor_t *base_des; @@ -665,8 +665,8 @@ void mca_btl_openib_endpoint_connected(mca_btl_openib_endpoint_t *endpoint) while(master && !opal_list_is_empty(&endpoint->ib_addr->pending_ep)) { ep_item = opal_list_remove_first(&endpoint->ib_addr->pending_ep); ep = (mca_btl_openib_endpoint_t *)ep_item; - if (OMPI_SUCCESS != - ompi_btl_openib_connect_base_start(endpoint->endpoint_local_cpc, + if (OMPI_SUCCESS != + ompi_btl_openib_connect_base_start(endpoint->endpoint_local_cpc, ep)) { BTL_ERROR(("Failed to connect pending endpoint\n")); } @@ -874,7 +874,7 @@ static int mca_btl_openib_endpoint_send_eager_rdma( rdma_hdr->control.type = MCA_BTL_OPENIB_CONTROL_RDMA; rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey; rdma_hdr->rdma_start.lval = ompi_ptr_ptol(endpoint->eager_rdma_local.base.pval); - BTL_VERBOSE(("sending rkey %" PRIu32 ", rdma_start.lval %" PRIx64 + BTL_VERBOSE(("sending rkey %" PRIu32 ", rdma_start.lval %" PRIx64 ", pval %p, ival %" PRIu32 " type %d and sizeof(rdma_hdr) %d\n", rdma_hdr->rkey, rdma_hdr->rdma_start.lval, diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.h b/ompi/mca/btl/openib/btl_openib_endpoint.h index 0f0b3ca96c..3fdef68f66 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.h +++ b/ompi/mca/btl/openib/btl_openib_endpoint.h @@ -208,7 +208,7 @@ struct mca_btl_base_endpoint_t { /** list of pending rget ops */ opal_list_t pending_get_frags; /** list of pending rput ops */ - opal_list_t pending_put_frags; + opal_list_t pending_put_frags; /** number of available get tokens */ int32_t get_tokens; @@ -503,8 +503,8 @@ static inline int post_send(mca_btl_openib_endpoint_t *ep, #if OPAL_ENABLE_DEBUG do { ftr->seq = ep->eager_rdma_remote.seq; - } while (!OPAL_ATOMIC_CMPSET_32((int32_t*) &ep->eager_rdma_remote.seq, - (int32_t) ftr->seq, + } while (!OPAL_ATOMIC_CMPSET_32((int32_t*) &ep->eager_rdma_remote.seq, + (int32_t) ftr->seq, (int32_t) (ftr->seq+1))); #endif if(ep->nbo) diff --git a/ompi/mca/btl/openib/btl_openib_failover.c b/ompi/mca/btl/openib/btl_openib_failover.c index 3359d59967..47c5ec5adc 100644 --- a/ompi/mca/btl/openib/btl_openib_failover.c +++ b/ompi/mca/btl/openib/btl_openib_failover.c @@ -56,7 +56,7 @@ static void dump_local_rdma_frags(mca_btl_openib_endpoint_t * endpoint); * @param qp Queue pair that had the error * @param remote_proc Pointer to process that had the error * @param endpoint Pointer to endpoint that had the error - */ + */ void mca_btl_openib_handle_endpoint_error(mca_btl_openib_module_t *openib_btl, mca_btl_base_descriptor_t *des, int qp, @@ -84,7 +84,7 @@ void mca_btl_openib_handle_endpoint_error(mca_btl_openib_module_t *openib_btl, * way to figure out what type of message created the error because * we need the information in the wc->imm_data field which does not * exist when we have an error. So, nothing to do here but return. */ - if ((openib_frag_type(des) == MCA_BTL_OPENIB_FRAG_RECV) && + if ((openib_frag_type(des) == MCA_BTL_OPENIB_FRAG_RECV) && !BTL_OPENIB_QP_TYPE_PP(qp)) { opal_output_verbose(20, mca_btl_openib_component.verbose_failover, "SRQ RECV type=%d", openib_frag_type(des)); @@ -108,7 +108,7 @@ void mca_btl_openib_handle_endpoint_error(mca_btl_openib_module_t *openib_btl, * B. It was some type of openib specific control message. * Therefore, just drop the fragments and call up into the PML to * disable this endpoint for future communication. */ - if (((openib_frag_type(des) == MCA_BTL_OPENIB_FRAG_RECV) && + if (((openib_frag_type(des) == MCA_BTL_OPENIB_FRAG_RECV) && (BTL_OPENIB_QP_TYPE_PP(qp))) || (openib_frag_type(des) == MCA_BTL_OPENIB_FRAG_CONTROL) || (openib_frag_type(des) == MCA_BTL_OPENIB_FRAG_EAGER_RDMA)) { @@ -244,7 +244,7 @@ void mca_btl_openib_handle_btl_error(mca_btl_openib_module_t* openib_btl) { NULL, btlname); if (NULL != btlname) free(btlname); - /* Now send out messages to all endpoints that we are disconnecting. + /* Now send out messages to all endpoints that we are disconnecting. * Only do this to endpoints that are connected. Otherwise, the * remote side does not yet have the information on this endpoint. */ for (i = 0; i < opal_pointer_array_get_size(openib_btl->device->endpoints); i++) { @@ -372,7 +372,7 @@ void btl_openib_handle_failover_control_messages(mca_btl_openib_control_header_t opal_output_verbose(20, mca_btl_openib_component.verbose_failover, "IB: rank=%d, control message (remote=%d), " "moved local head by one (new=%d)", - ORTE_PROC_MY_NAME->vpid, + ORTE_PROC_MY_NAME->vpid, newep->endpoint_proc->proc_ompi->proc_name.vpid, newep->eager_rdma_local.head); } else { @@ -399,9 +399,9 @@ void btl_openib_handle_failover_control_messages(mca_btl_openib_control_header_t * and call the callback function with OMPI_ERROR. It walks through * each qp with each priority and looks for both no_credits_pending_frags * and no_wqe_pending_frags. It then looks for any pending_lazy_frags, - * pending_put_frags, and pending_get_frags. This function is only + * pending_put_frags, and pending_get_frags. This function is only * called when running with failover support enabled. Note that - * the errout parameter allows the function to also be used as a + * the errout parameter allows the function to also be used as a * debugging tool to see if there are any fragments on any of the * queues. * @param ep Pointer to endpoint that had error @@ -753,7 +753,7 @@ void mca_btl_openib_dump_all_local_rdma_frags(mca_btl_openib_device_t *device) { /** * This function is a debugging tool. If you notify a hang, you can - * call this function from a debugger and see if there are any + * call this function from a debugger and see if there are any * messages stuck in any of the queues. If you call it with * errout=true, then it will error them out. Otherwise, it will * just print out the size of the queues with data in them. @@ -769,7 +769,7 @@ void mca_btl_openib_dump_all_internal_queues(bool errout) { btl = mca_btl_openib_component.openib_btls[i]; module = &btl->super; num_eps = opal_pointer_array_get_size(btl->device->endpoints); - + /* Now, find the endpoint associated with it */ for (j = 0; j < num_eps; j++) { ep = (mca_btl_openib_endpoint_t*) diff --git a/ompi/mca/btl/openib/btl_openib_fd.c b/ompi/mca/btl/openib/btl_openib_fd.c index 788b308527..7147373b9c 100644 --- a/ompi/mca/btl/openib/btl_openib_fd.c +++ b/ompi/mca/btl/openib/btl_openib_fd.c @@ -3,9 +3,9 @@ * Copyright (c) 2009 Sandia National Laboratories. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -174,7 +174,7 @@ static int service_pipe_cmd_add_fd(bool use_libevent, cmd_t *cmd) if (use_libevent) { /* Make an event for this fd */ ri->ri_event_used = true; - opal_event_set(opal_event_base, &ri->ri_event, ri->ri_fd, + opal_event_set(opal_event_base, &ri->ri_event, ri->ri_fd, ri->ri_flags | OPAL_EV_PERSIST, service_fd_callback, ri); opal_event_add(&ri->ri_event, 0); @@ -248,13 +248,13 @@ static int service_pipe_cmd_remove_fd(cmd_t *cmd) } } } - + /* Let the caller know that we have stopped monitoring this fd (if they care) */ if (NULL != cmd->pc_fn.event) { cmd->pc_fn.event(cmd->pc_fd, 0, cmd->pc_context); } - + /* Remove this item from the list of registered items and release it */ opal_list_remove_item(®istered_items, item); @@ -344,7 +344,7 @@ static bool service_pipe_cmd(void) --waiting_for_ack_from_main_thread; } break; - + default: OPAL_OUTPUT((-1, "fd service thread: unknown pipe command!")); break; @@ -390,7 +390,7 @@ static void *service_thread_start(void *context) break; } OPAL_OUTPUT((-1, "fd service thread: back from pipe command")); - } + } /* Go through all the registered events and see who had activity */ @@ -414,7 +414,7 @@ static void *service_thread_start(void *context) /* If either was ready, invoke the callback */ if (0 != flags) { OPAL_OUTPUT((-1, "fd service thread: invoking callback for registered fd %d", ri->ri_fd)); - ri->ri_callback.event(ri->ri_fd, flags, + ri->ri_callback.event(ri->ri_fd, flags, ri->ri_context); OPAL_OUTPUT((-1, "fd service thread: back from callback for registered fd %d", ri->ri_fd)); } @@ -443,7 +443,7 @@ static void main_thread_event_callback(int fd, short event, void *context) break; default: - OPAL_OUTPUT((-1, "fd main thread: unknown pipe command: %d", + OPAL_OUTPUT((-1, "fd main thread: unknown pipe command: %d", cmd.pc_cmd)); break; } @@ -481,12 +481,12 @@ int ompi_btl_openib_fd_init(void) /* Create a libevent event that is used in the main thread to watch its pipe */ opal_event_set(opal_event_base, &main_thread_event, pipe_to_main_thread[0], - OPAL_EV_READ | OPAL_EV_PERSIST, + OPAL_EV_READ | OPAL_EV_PERSIST, main_thread_event_callback, NULL); opal_event_add(&main_thread_event, 0); - + /* Start the service thread */ - if (0 != pthread_create(&thread, NULL, service_thread_start, + if (0 != pthread_create(&thread, NULL, service_thread_start, NULL)) { int errno_save = errno; opal_event_del(&main_thread_event); @@ -509,7 +509,7 @@ int ompi_btl_openib_fd_init(void) * Start monitoring an fd * Called by main or service thread; callback will be in service thread */ -int ompi_btl_openib_fd_monitor(int fd, int flags, +int ompi_btl_openib_fd_monitor(int fd, int flags, ompi_btl_openib_fd_event_callback_fn_t *callback, void *context) { @@ -542,7 +542,7 @@ int ompi_btl_openib_fd_monitor(int fd, int flags, * Stop monitoring an fd * Called by main or service thread; callback will be in service thread */ -int ompi_btl_openib_fd_unmonitor(int fd, +int ompi_btl_openib_fd_unmonitor(int fd, ompi_btl_openib_fd_event_callback_fn_t *callback, void *context) { @@ -552,7 +552,7 @@ int ompi_btl_openib_fd_unmonitor(int fd, if (fd < 0) { return OMPI_ERR_BAD_PARAM; } - + cmd.pc_cmd = CMD_REMOVE_FD; cmd.pc_fd = fd; cmd.pc_flags = 0; @@ -630,7 +630,7 @@ ompi_btl_openib_fd_main_thread_drain(void) int nfds, ret; fd_set rfds; struct timeval tv; - + while (1) { FD_ZERO(&rfds); FD_SET(pipe_to_main_thread[0], &rfds); @@ -665,10 +665,10 @@ int ompi_btl_openib_fd_finalize(void) memset(&cmd, 0, cmd_size); cmd.pc_cmd = CMD_TIME_TO_QUIT; opal_fd_write(pipe_to_service_thread[1], cmd_size, &cmd); - + pthread_join(thread, NULL); opal_atomic_rmb(); - + opal_event_del(&main_thread_event); close(pipe_to_service_thread[0]); diff --git a/ompi/mca/btl/openib/btl_openib_fd.h b/ompi/mca/btl/openib/btl_openib_fd.h index b549063a18..b4880e995d 100644 --- a/ompi/mca/btl/openib/btl_openib_fd.h +++ b/ompi/mca/btl/openib/btl_openib_fd.h @@ -3,9 +3,9 @@ * Copyright (c) 2009 Sandia National Laboratories. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -19,7 +19,7 @@ BEGIN_C_DECLS /** * Typedef for fd callback function */ -typedef void *(ompi_btl_openib_fd_event_callback_fn_t)(int fd, int flags, +typedef void *(ompi_btl_openib_fd_event_callback_fn_t)(int fd, int flags, void *context); /** @@ -37,7 +37,7 @@ int ompi_btl_openib_fd_init(void); * Start monitoring an fd. * Called by main or service thread; callback will be in service thread. */ -int ompi_btl_openib_fd_monitor(int fd, int flags, +int ompi_btl_openib_fd_monitor(int fd, int flags, ompi_btl_openib_fd_event_callback_fn_t *callback, void *context); @@ -45,7 +45,7 @@ int ompi_btl_openib_fd_monitor(int fd, int flags, * Stop monitoring an fd. * Called by main or service thread; callback will be in service thread. */ -int ompi_btl_openib_fd_unmonitor(int fd, +int ompi_btl_openib_fd_unmonitor(int fd, ompi_btl_openib_fd_event_callback_fn_t *callback, void *context); diff --git a/ompi/mca/btl/openib/btl_openib_ini.c b/ompi/mca/btl/openib/btl_openib_ini.c index a3ee801fb5..b9d0a89df5 100644 --- a/ompi/mca/btl/openib/btl_openib_ini.c +++ b/ompi/mca/btl/openib/btl_openib_ini.c @@ -110,7 +110,7 @@ int ompi_btl_openib_ini_init(void) #ifndef __WINDOWS__ char separator = ':'; #else - /* ':' is part of the path on Windows, + /* ':' is part of the path on Windows, so use ';' instead. */ char separator = ';'; #endif @@ -414,7 +414,7 @@ static int parse_line(parsed_section_values_t *sv) else if (0 == strcasecmp(key_buffer, "rdmacm_reject_causes_connect_error")) { /* Single value */ - sv->values.rdmacm_reject_causes_connect_error = + sv->values.rdmacm_reject_causes_connect_error = (bool) ompi_btl_openib_ini_intify(value); sv->values.rdmacm_reject_causes_connect_error_set = true; } @@ -558,7 +558,7 @@ static int save_section(parsed_section_values_t *s) } if (NULL != s->values.receive_queues) { - h->values.receive_queues = + h->values.receive_queues = strdup(s->values.receive_queues); } @@ -568,9 +568,9 @@ static int save_section(parsed_section_values_t *s) } if (s->values.rdmacm_reject_causes_connect_error_set) { - h->values.rdmacm_reject_causes_connect_error = + h->values.rdmacm_reject_causes_connect_error = s->values.rdmacm_reject_causes_connect_error; - h->values.rdmacm_reject_causes_connect_error_set = + h->values.rdmacm_reject_causes_connect_error_set = true; } diff --git a/ompi/mca/btl/openib/btl_openib_ip.c b/ompi/mca/btl/openib/btl_openib_ip.c index 01fc3a0f94..ebcde0b2a6 100644 --- a/ompi/mca/btl/openib/btl_openib_ip.c +++ b/ompi/mca/btl/openib/btl_openib_ip.c @@ -31,9 +31,9 @@ #include "btl_openib_ip.h" #if OMPI_HAVE_RDMACM -/* +/* * The cruft below maintains the linked list of rdma ipv4 addresses and their - * associated rdma device names and device port numbers. + * associated rdma device names and device port numbers. */ struct rdma_addr_list { opal_list_item_t super; @@ -45,7 +45,7 @@ struct rdma_addr_list { }; typedef struct rdma_addr_list rdma_addr_list_t; -static OBJ_CLASS_INSTANCE(rdma_addr_list_t, opal_list_item_t, +static OBJ_CLASS_INSTANCE(rdma_addr_list_t, opal_list_item_t, NULL, NULL); static opal_list_t *myaddrs = NULL; @@ -54,7 +54,7 @@ static char *stringify(uint32_t addr) { static char line[64]; memset(line, 0, sizeof(line)); - snprintf(line, sizeof(line) - 1, "%d.%d.%d.%d (0x%x)", + snprintf(line, sizeof(line) - 1, "%d.%d.%d.%d (0x%x)", #if defined(WORDS_BIGENDIAN) (addr >> 24), (addr >> 16) & 0xff, @@ -119,7 +119,7 @@ uint64_t mca_btl_openib_get_ip_subnet_id(struct ibv_device *ib_dev, * mismatch if IP Aliases are being used. For more information on * this, please read comment above mca_btl_openib_get_ip_subnet_id. */ -uint32_t mca_btl_openib_rdma_get_ipv4addr(struct ibv_context *verbs, +uint32_t mca_btl_openib_rdma_get_ipv4addr(struct ibv_context *verbs, uint8_t port) { opal_list_item_t *item; @@ -135,7 +135,7 @@ uint32_t mca_btl_openib_rdma_get_ipv4addr(struct ibv_context *verbs, item != opal_list_get_end(myaddrs); item = opal_list_get_next(item)) { struct rdma_addr_list *addr = (struct rdma_addr_list *)item; - if (!strcmp(addr->dev_name, verbs->device->name) && + if (!strcmp(addr->dev_name, verbs->device->name) && port == addr->dev_port) { BTL_VERBOSE(("FOUND: %s:%d is %s", ibv_get_device_name(verbs->device), port, @@ -219,7 +219,7 @@ static int ipaddr_specified(struct sockaddr_in *ipaddr, uint32_t netmask) subnet = ntohl(ipaddr->sin_addr.s_addr) & ~(all >> netmask); opal_argv_free(temp); - if (subnet == list_subnet) { + if (subnet == list_subnet) { return 0; } } @@ -261,7 +261,7 @@ static int ipaddr_specified(struct sockaddr_in *ipaddr, uint32_t netmask) subnet = ntohl(ipaddr->sin_addr.s_addr) & ~(all >> netmask); opal_argv_free(temp); - if (subnet == list_subnet) { + if (subnet == list_subnet) { return 1; } } @@ -282,7 +282,7 @@ static int add_rdma_addr(struct sockaddr *ipaddr, uint32_t netmask) /* Ensure that this IP address is not in 127.0.0.1/8. If it is, skip it because we never want loopback addresses to be considered RDMA devices that remote peers can use to connect - to. + to. This check is necessary because of a change that almost went into RDMA CM in OFED 1.5.1. We asked for a delay so that we @@ -356,11 +356,11 @@ static int add_rdma_addr(struct sockaddr *ipaddr, uint32_t netmask) myaddr->addr = sinp->sin_addr.s_addr; myaddr->subnet = ntohl(myaddr->addr) & ~(all >> netmask); - inet_ntop(sinp->sin_family, &sinp->sin_addr, + inet_ntop(sinp->sin_family, &sinp->sin_addr, myaddr->addr_str, sizeof(myaddr->addr_str)); memcpy(myaddr->dev_name, cm_id->verbs->device->name, IBV_SYSFS_NAME_MAX); myaddr->dev_port = cm_id->port_num; - BTL_VERBOSE(("Adding addr %s (0x%x) subnet 0x%x as %s:%d", + BTL_VERBOSE(("Adding addr %s (0x%x) subnet 0x%x as %s:%d", myaddr->addr_str, myaddr->addr, myaddr->subnet, myaddr->dev_name, myaddr->dev_port)); @@ -400,7 +400,7 @@ int mca_btl_openib_build_rdma_addr_list(void) } return rc; } - + void mca_btl_openib_free_rdma_addr_list(void) { opal_list_item_t *item, *next; @@ -419,27 +419,27 @@ void mca_btl_openib_free_rdma_addr_list(void) } } -#else +#else /* !OMPI_HAVE_RDMACM case */ uint64_t mca_btl_openib_get_ip_subnet_id(struct ibv_device *ib_dev, - uint8_t port) + uint8_t port) { return 0; } -uint32_t mca_btl_openib_rdma_get_ipv4addr(struct ibv_context *verbs, - uint8_t port) +uint32_t mca_btl_openib_rdma_get_ipv4addr(struct ibv_context *verbs, + uint8_t port) { return 0; } -int mca_btl_openib_build_rdma_addr_list(void) +int mca_btl_openib_build_rdma_addr_list(void) { return OMPI_SUCCESS; } -void mca_btl_openib_free_rdma_addr_list(void) +void mca_btl_openib_free_rdma_addr_list(void) { } #endif diff --git a/ompi/mca/btl/openib/btl_openib_ip.h b/ompi/mca/btl/openib/btl_openib_ip.h index 90a0da974b..5ad5b5d86a 100644 --- a/ompi/mca/btl/openib/btl_openib_ip.h +++ b/ompi/mca/btl/openib/btl_openib_ip.h @@ -32,7 +32,7 @@ extern uint64_t mca_btl_openib_get_ip_subnet_id(struct ibv_device *ib_dev, * @param port (IN) physical port of the IBV device * @return IPv4 Address */ -extern uint32_t mca_btl_openib_rdma_get_ipv4addr(struct ibv_context *verbs, +extern uint32_t mca_btl_openib_rdma_get_ipv4addr(struct ibv_context *verbs, uint8_t port); /** diff --git a/ompi/mca/btl/openib/btl_openib_lex.l b/ompi/mca/btl/openib/btl_openib_lex.l index e50db26016..0e3bdbdeb0 100644 --- a/ompi/mca/btl/openib/btl_openib_lex.l +++ b/ompi/mca/btl/openib/btl_openib_lex.l @@ -6,15 +6,15 @@ * Copyright (c) 2004-2005 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -72,13 +72,13 @@ NAME_CHAR [A-Za-z0-9_\-\.\\\/] [^*\n]* ; /* Eat up non '*'s */ "*"+[^*/\n]* ; /* Eat '*'s not followed by a '/' */ \n { ++btl_openib_ini_yynewlines; - return BTL_OPENIB_INI_PARSE_NEWLINE; } + return BTL_OPENIB_INI_PARSE_NEWLINE; } "*"+"/" { BEGIN(INITIAL); /* Done with block comment */ return BTL_OPENIB_INI_PARSE_NEWLINE; } {WHITE}*\[{WHITE}* { BEGIN(section_name); } ({NAME_CHAR}|{WHITE})*{NAME_CHAR}/{WHITE}*\] { - BEGIN(section_end); + BEGIN(section_end); return BTL_OPENIB_INI_PARSE_SECTION; } \n { ++btl_openib_ini_yynewlines; return BTL_OPENIB_INI_PARSE_ERROR; } @@ -87,7 +87,7 @@ NAME_CHAR [A-Za-z0-9_\-\.\\\/] ++btl_openib_ini_yynewlines; return BTL_OPENIB_INI_PARSE_NEWLINE; } -{WHITE}*"="{WHITE}* { BEGIN(value); +{WHITE}*"="{WHITE}* { BEGIN(value); return BTL_OPENIB_INI_PARSE_EQUAL; } {WHITE}+ ; /* whitespace */ {CHAR}+ { return BTL_OPENIB_INI_PARSE_SINGLE_WORD; } @@ -95,7 +95,7 @@ NAME_CHAR [A-Za-z0-9_\-\.\\\/] {WHITE}*\n { BEGIN(INITIAL); ++btl_openib_ini_yynewlines; return BTL_OPENIB_INI_PARSE_NEWLINE; } -[^\n]*[^\t \n]/[\t ]* { +[^\n]*[^\t \n]/[\t ]* { return BTL_OPENIB_INI_PARSE_VALUE; } . { return BTL_OPENIB_INI_PARSE_ERROR; } @@ -107,10 +107,10 @@ NAME_CHAR [A-Za-z0-9_\-\.\\\/] * This cleans up at the end of the parse (since, in this case, we * always parse the entire file) and prevents a memory leak. */ -static int finish_parsing(void) +static int finish_parsing(void) { if (NULL != YY_CURRENT_BUFFER) { - yy_delete_buffer(YY_CURRENT_BUFFER); + yy_delete_buffer(YY_CURRENT_BUFFER); #if defined(YY_CURRENT_BUFFER_LVALUE) YY_CURRENT_BUFFER_LVALUE = NULL; #else diff --git a/ompi/mca/btl/openib/btl_openib_mca.c b/ompi/mca/btl/openib/btl_openib_mca.c index 48722486e6..19f0a41f5f 100644 --- a/ompi/mca/btl/openib/btl_openib_mca.c +++ b/ompi/mca/btl/openib/btl_openib_mca.c @@ -64,7 +64,7 @@ enum { /* * utility routine for string parameter registration */ -static int reg_string(const char* param_name, +static int reg_string(const char* param_name, const char* deprecated_param_name, const char* param_desc, const char* default_value, char **out_value, @@ -76,8 +76,8 @@ static int reg_string(const char* param_name, param_name, param_desc, false, false, default_value, &value); if (NULL != deprecated_param_name) { - mca_base_param_reg_syn(index, - &mca_btl_openib_component.super.btl_version, + mca_base_param_reg_syn(index, + &mca_btl_openib_component.super.btl_version, deprecated_param_name, true); } mca_base_param_lookup_string(index, &value); @@ -95,7 +95,7 @@ static int reg_string(const char* param_name, /* * utility routine for integer parameter registration */ -static int reg_int(const char* param_name, +static int reg_int(const char* param_name, const char* deprecated_param_name, const char* param_desc, int default_value, int *out_value, int flags) @@ -105,12 +105,12 @@ static int reg_int(const char* param_name, param_name, param_desc, false, false, default_value, NULL); if (NULL != deprecated_param_name) { - mca_base_param_reg_syn(index, - &mca_btl_openib_component.super.btl_version, + mca_base_param_reg_syn(index, + &mca_btl_openib_component.super.btl_version, deprecated_param_name, true); } mca_base_param_lookup_int(index, &value); - + if (0 != (flags & REGINT_NEG_ONE_OK) && -1 == value) { *out_value = value; return OMPI_SUCCESS; @@ -193,7 +193,7 @@ int btl_openib_register_mca_params(void) } CHECK(reg_string("device_param_files", "hca_param_files", "Colon-delimited list of INI-style files that contain device vendor/part-specific parameters (use semicolon for Windows)", - str, &mca_btl_openib_component.device_params_file_names, + str, &mca_btl_openib_component.device_params_file_names, 0)); free(str); @@ -264,11 +264,11 @@ int btl_openib_register_mca_params(void) -1, &ival, REGINT_NEG_ONE_OK | REGINT_GE_ZERO)); mca_btl_openib_component.ib_max_inline_data = (int32_t) ival; - CHECK(reg_string("pkey", "ib_pkey_val", + CHECK(reg_string("pkey", "ib_pkey_val", "OpenFabrics partition key (pkey) value. " "Unsigned integer decimal or hex values are allowed (e.g., \"3\" or \"0x3f\") and will be masked against the maximum allowable IB partition key value (0x7fff)", "0", &pkey, 0)); - mca_btl_openib_component.ib_pkey_val = + mca_btl_openib_component.ib_pkey_val = ompi_btl_openib_ini_intify(pkey) & MCA_BTL_IB_PKEY_MASK; free(pkey); @@ -278,7 +278,7 @@ int btl_openib_register_mca_params(void) 0, &ival, REGINT_GE_ZERO)); mca_btl_openib_component.ib_psn = (uint32_t) ival; - CHECK(reg_int("ib_qp_ous_rd_atom", NULL, + CHECK(reg_int("ib_qp_ous_rd_atom", NULL, "InfiniBand outstanding atomic reads " "(must be >= 0)", 4, &ival, REGINT_GE_ZERO)); @@ -402,7 +402,7 @@ int btl_openib_register_mca_params(void) CHECK(reg_int("ib_path_record_service_level", NULL, "Enable getting InfiniBand service level from PathRecord " "(must be >= 0, 0 = disabled, positive = try to get the " - "service level from PathRecord)", + "service level from PathRecord)", 0, &ival, REGINT_GE_ZERO)); mca_btl_openib_component.ib_path_record_service_level = (uint32_t) ival; #endif @@ -582,11 +582,11 @@ int btl_openib_register_mca_params(void) CHECK(reg_string("receive_queues", NULL, "Colon-delimited, comma-delimited list of receive queues: P,4096,8,6,4:P,32768,8,6,4", - default_qps, &mca_btl_openib_component.receive_queues, + default_qps, &mca_btl_openib_component.receive_queues, 0)); - mca_btl_openib_component.receive_queues_source = - (0 == strcmp(default_qps, - mca_btl_openib_component.receive_queues)) ? + mca_btl_openib_component.receive_queues_source = + (0 == strcmp(default_qps, + mca_btl_openib_component.receive_queues)) ? BTL_OPENIB_RQ_SOURCE_DEFAULT : BTL_OPENIB_RQ_SOURCE_MCA; CHECK(reg_string("if_include", NULL, diff --git a/ompi/mca/btl/openib/btl_openib_proc.c b/ompi/mca/btl/openib/btl_openib_proc.c index ea4aa4bc90..79b6a8b983 100644 --- a/ompi/mca/btl/openib/btl_openib_proc.c +++ b/ompi/mca/btl/openib/btl_openib_proc.c @@ -172,7 +172,7 @@ mca_btl_openib_proc_t* mca_btl_openib_proc_create(ompi_proc_t* ompi_proc) BTL_VERBOSE(("unpack: %d btls", module_proc->proc_port_count)); if (module_proc->proc_port_count > 0) { module_proc->proc_ports = (mca_btl_openib_proc_modex_t *) - malloc(sizeof(mca_btl_openib_proc_modex_t) * + malloc(sizeof(mca_btl_openib_proc_modex_t) * module_proc->proc_port_count); } else { module_proc->proc_ports = NULL; @@ -194,7 +194,7 @@ mca_btl_openib_proc_t* mca_btl_openib_proc_create(ompi_proc_t* ompi_proc) /* Unpack the number of CPCs that follow */ unpack8(&offset, &(module_proc->proc_ports[i].pm_cpc_data_count)); BTL_VERBOSE(("unpacked btl %d: number of cpcs to follow %d (offset now %d)", - i, module_proc->proc_ports[i].pm_cpc_data_count, + i, module_proc->proc_ports[i].pm_cpc_data_count, (int)(offset-((char*)message)))); module_proc->proc_ports[i].pm_cpc_data = (ompi_btl_openib_connect_base_module_data_t *) calloc(module_proc->proc_ports[i].pm_cpc_data_count, @@ -211,15 +211,15 @@ mca_btl_openib_proc_t* mca_btl_openib_proc_create(ompi_proc_t* ompi_proc) unpack8(&offset, &u8); BTL_VERBOSE(("unpacked btl %d: cpc %d: index %d (offset now %d)", i, j, u8, (int)(offset-(char*)message))); - cpcd->cbm_component = + cpcd->cbm_component = ompi_btl_openib_connect_base_get_cpc_byindex(u8); BTL_VERBOSE(("unpacked btl %d: cpc %d: component %s", i, j, cpcd->cbm_component->cbc_name)); - + unpack8(&offset, &cpcd->cbm_priority); unpack8(&offset, &cpcd->cbm_modex_message_len); BTL_VERBOSE(("unpacked btl %d: cpc %d: priority %d, msg len %d (offset now %d)", - i, j, cpcd->cbm_priority, + i, j, cpcd->cbm_priority, cpcd->cbm_modex_message_len, (int)(offset-(char*)message))); if (cpcd->cbm_modex_message_len > 0) { @@ -228,7 +228,7 @@ mca_btl_openib_proc_t* mca_btl_openib_proc_create(ompi_proc_t* ompi_proc) BTL_ERROR(("Failed to malloc")); return NULL; } - memcpy(cpcd->cbm_modex_message, offset, + memcpy(cpcd->cbm_modex_message, offset, cpcd->cbm_modex_message_len); offset += cpcd->cbm_modex_message_len; BTL_VERBOSE(("unpacked btl %d: cpc %d: blob unpacked %d %x (offset now %d)", @@ -244,7 +244,7 @@ mca_btl_openib_proc_t* mca_btl_openib_proc_create(ompi_proc_t* ompi_proc) module_proc->proc_endpoints = NULL; } else { module_proc->proc_endpoints = (mca_btl_base_endpoint_t**) - malloc(module_proc->proc_port_count * + malloc(module_proc->proc_port_count * sizeof(mca_btl_base_endpoint_t*)); } if (NULL == module_proc->proc_endpoints) { diff --git a/ompi/mca/btl/openib/btl_openib_proc.h b/ompi/mca/btl/openib/btl_openib_proc.h index 203d5c8cf8..8f5984edaf 100644 --- a/ompi/mca/btl/openib/btl_openib_proc.h +++ b/ompi/mca/btl/openib/btl_openib_proc.h @@ -47,7 +47,7 @@ typedef struct mca_btl_openib_proc_modex_t { /** Array of the peer's CPCs available on this port */ ompi_btl_openib_connect_base_module_data_t *pm_cpc_data; - + /** Length of the pm_cpc_data array */ uint8_t pm_cpc_data_count; } mca_btl_openib_proc_modex_t; diff --git a/ompi/mca/btl/openib/configure.m4 b/ompi/mca/btl/openib/configure.m4 index a577ee2372..afcfa39009 100644 --- a/ompi/mca/btl/openib/configure.m4 +++ b/ompi/mca/btl/openib/configure.m4 @@ -6,7 +6,7 @@ # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. @@ -14,9 +14,9 @@ # Copyright (c) 2008 Mellanox Technologies. All rights reserved. # Copyright (c) 2011 Oracle and/or its affiliates. All rights reserved. # $COPYRIGHT$ -# +# # Additional copyrights may follow -# +# # $HEADER$ # @@ -29,7 +29,7 @@ AC_DEFUN([MCA_ompi_btl_openib_POST_CONFIG], [ ]) -# MCA_btl_openib_CONFIG([action-if-can-compile], +# MCA_btl_openib_CONFIG([action-if-can-compile], # [action-if-cant-compile]) # ------------------------------------------------ AC_DEFUN([MCA_ompi_btl_openib_CONFIG],[ diff --git a/ompi/mca/btl/openib/connect/base.h b/ompi/mca/btl/openib/connect/base.h index 5952943f33..b4de20304d 100644 --- a/ompi/mca/btl/openib/connect/base.h +++ b/ompi/mca/btl/openib/connect/base.h @@ -3,9 +3,9 @@ * Copyright (c) 2009 Mellanox Technologies. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -74,13 +74,13 @@ int ompi_btl_openib_connect_base_get_cpc_index ompi_btl_openib_connect_base_component_t * ompi_btl_openib_connect_base_get_cpc_byindex(uint8_t index); -/* +/* * Allocate a CTS frag */ int ompi_btl_openib_connect_base_alloc_cts( struct mca_btl_base_endpoint_t *endpoint); -/* +/* * Free a CTS frag */ int ompi_btl_openib_connect_base_free_cts( diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_base.c b/ompi/mca/btl/openib/connect/btl_openib_connect_base.c index b7778c420e..09d39793f5 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_base.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_base.c @@ -3,9 +3,9 @@ * Copyright (c) 2007 Mellanox Technologies, Inc. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ #include "ompi_config.h" @@ -88,7 +88,7 @@ int ompi_btl_openib_connect_base_register(void) all_cpc_names); mca_base_param_reg_string(&mca_btl_openib_component.super.btl_version, - "cpc_include", string, false, false, + "cpc_include", string, false, false, NULL, &cpc_include); free(string); @@ -97,7 +97,7 @@ int ompi_btl_openib_connect_base_register(void) all_cpc_names); mca_base_param_reg_string(&mca_btl_openib_component.super.btl_version, - "cpc_exclude", string, false, false, + "cpc_exclude", string, false, false, NULL, &cpc_exclude); free(string); @@ -112,7 +112,7 @@ int ompi_btl_openib_connect_base_register(void) temp = opal_argv_split(cpc_include, ','); for (save = j = 0; NULL != temp[j]; ++j) { for (i = 0; NULL != all[i]; ++i) { - if (0 == strcmp(temp[j], all[i]->cbc_name)) { + if (0 == strcmp(temp[j], all[i]->cbc_name)) { opal_output(-1, "include: saving %s", all[i]->cbc_name); available[save++] = all[i]; ++num_available; @@ -123,7 +123,7 @@ int ompi_btl_openib_connect_base_register(void) orte_show_help("help-mpi-btl-openib-cpc-base.txt", "cpc name not found", true, "include", orte_process_info.nodename, - "include", cpc_include, temp[j], + "include", cpc_include, temp[j], all_cpc_names); opal_argv_free(temp); free(all_cpc_names); @@ -141,7 +141,7 @@ int ompi_btl_openib_connect_base_register(void) /* First: error check -- ensure that all the names are valid */ for (j = 0; NULL != temp[j]; ++j) { for (i = 0; NULL != all[i]; ++i) { - if (0 == strcmp(temp[j], all[i]->cbc_name)) { + if (0 == strcmp(temp[j], all[i]->cbc_name)) { break; } } @@ -149,7 +149,7 @@ int ompi_btl_openib_connect_base_register(void) orte_show_help("help-mpi-btl-openib-cpc-base.txt", "cpc name not found", true, "exclude", orte_process_info.nodename, - "exclude", cpc_exclude, temp[j], + "exclude", cpc_exclude, temp[j], all_cpc_names); opal_argv_free(temp); free(all_cpc_names); @@ -171,13 +171,13 @@ int ompi_btl_openib_connect_base_register(void) } } opal_argv_free(temp); - } + } /* If there's no include/exclude list, copy all[] into available[] */ else { opal_output(-1, "no include or exclude: saving all"); memcpy(available, all, sizeof(all)); - num_available = (sizeof(all) / + num_available = (sizeof(all) / sizeof(ompi_btl_openib_connect_base_module_t *)) - 1; } @@ -241,7 +241,7 @@ int ompi_btl_openib_connect_base_select_for_local_port(mca_btl_openib_module_t * int i, rc, cpc_index, len; ompi_btl_openib_connect_base_module_t **cpcs; - cpcs = (ompi_btl_openib_connect_base_module_t **) calloc(num_available, + cpcs = (ompi_btl_openib_connect_base_module_t **) calloc(num_available, sizeof(ompi_btl_openib_connect_base_module_t *)); if (NULL == cpcs) { return OMPI_ERR_OUT_OF_RESOURCE; @@ -311,7 +311,7 @@ int ompi_btl_openib_connect_base_select_for_local_port(mca_btl_openib_module_t * return OMPI_SUCCESS; } -/* +/* * This function is invoked when determining whether we have a CPC in * common with a specific remote port. We already know that the * subnet ID is the same between a specific local port and the target @@ -398,7 +398,7 @@ int ompi_btl_openib_connect_base_get_cpc_index(ompi_btl_openib_connect_base_comp ompi_btl_openib_connect_base_component_t * ompi_btl_openib_connect_base_get_cpc_byindex(uint8_t index) { - return (index >= (sizeof(all) / + return (index >= (sizeof(all) / sizeof(ompi_btl_openib_connect_base_module_t *))) ? NULL : all[index]; } @@ -421,8 +421,8 @@ int ompi_btl_openib_connect_base_alloc_cts(mca_btl_base_endpoint_t *endpoint) return OMPI_ERR_OUT_OF_RESOURCE; } - endpoint->endpoint_cts_mr = - ibv_reg_mr(endpoint->endpoint_btl->device->ib_pd, + endpoint->endpoint_cts_mr = + ibv_reg_mr(endpoint->endpoint_btl->device->ib_pd, fli->ptr, length, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ); @@ -438,14 +438,14 @@ int ompi_btl_openib_connect_base_alloc_cts(mca_btl_base_endpoint_t *endpoint) from underneath us. */ /* Copy the lkey where it needs to go */ - endpoint->endpoint_cts_frag.super.sg_entry.lkey = - endpoint->endpoint_cts_frag.super.super.segment.seg_key.key32[0] = + endpoint->endpoint_cts_frag.super.sg_entry.lkey = + endpoint->endpoint_cts_frag.super.super.segment.seg_key.key32[0] = endpoint->endpoint_cts_mr->lkey; endpoint->endpoint_cts_frag.super.sg_entry.length = length; /* Construct the rest of the recv_frag_t */ OBJ_CONSTRUCT(&(endpoint->endpoint_cts_frag), mca_btl_openib_recv_frag_t); - endpoint->endpoint_cts_frag.super.super.base.order = + endpoint->endpoint_cts_frag.super.super.base.order = mca_btl_openib_component.credits_qp; endpoint->endpoint_cts_frag.super.endpoint = endpoint; OPAL_OUTPUT((-1, "Got a CTS frag for peer %s, addr %p, length %d, lkey %d", diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_empty.c b/ompi/mca/btl/openib/connect/btl_openib_connect_empty.c index 23f5844bb7..8795e13b37 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_empty.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_empty.c @@ -2,9 +2,9 @@ * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -16,7 +16,7 @@ static void empty_component_register(void); static int empty_component_init(void); -static int empty_component_query(mca_btl_openib_module_t *btl, +static int empty_component_query(mca_btl_openib_module_t *btl, ompi_btl_openib_connect_base_module_t **cpc); ompi_btl_openib_connect_base_component_t ompi_btl_openib_connect_empty = { @@ -38,7 +38,7 @@ static int empty_component_init(void) return OMPI_ERR_NOT_SUPPORTED; } -static int empty_component_query(mca_btl_openib_module_t *btl, +static int empty_component_query(mca_btl_openib_module_t *btl, ompi_btl_openib_connect_base_module_t **cpc) { /* Never let this CPC run */ diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_empty.h b/ompi/mca/btl/openib/connect/btl_openib_connect_empty.h index 8a27e499b2..01a24e44d1 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_empty.h +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_empty.h @@ -2,9 +2,9 @@ * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c index 62f6a3fab7..23012bc330 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.c @@ -4,9 +4,9 @@ * Copyright (c) 2009 IBM Corporation. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -14,7 +14,7 @@ * TO-DO: * * - audit control values passed to req_send() - * - More show_help() throughout + * - More show_help() throughout * - error handling in case of broken connection is not good; need to * notify btl module safely */ @@ -91,7 +91,7 @@ * tell IBCM (among other things) when the first message arrives on * a QP when the RTU has not yet been received. This can happen, of * course, since IBCM traffic is UD. - * - Also, note that IBCM "listener" IDs are per DEVICE, not per port. + * - Also, note that IBCM "listener" IDs are per DEVICE, not per port. * - CM ID's are persistent throughout the life of a QP. If you * destroy a CM ID (ib_cm_destroy_id), the IBCM system will tear * down the connection. So the CM ID you get when receiving a @@ -179,7 +179,7 @@ * * 1. The "wrong" process will send a single IBCM connection request * to its peer on a bogus QP that was created just for this - * request. + * request. * 2. The receiver will get the request, detect that it came * in from the "wrong" direction, and reject it (IBCM has an * explicit provision for rejecting incoming connections). @@ -366,7 +366,7 @@ typedef struct { static void ibcm_listen_cm_id_constructor(ibcm_listen_cm_id_t *h); static void ibcm_listen_cm_id_destructor(ibcm_listen_cm_id_t *h); -static OBJ_CLASS_INSTANCE(ibcm_listen_cm_id_t, opal_list_item_t, +static OBJ_CLASS_INSTANCE(ibcm_listen_cm_id_t, opal_list_item_t, ibcm_listen_cm_id_constructor, ibcm_listen_cm_id_destructor); @@ -452,7 +452,7 @@ typedef struct { ibcm_module_t *ibcm_module; } ibcm_module_list_item_t; -static OBJ_CLASS_INSTANCE(ibcm_module_list_item_t, opal_list_item_t, +static OBJ_CLASS_INSTANCE(ibcm_module_list_item_t, opal_list_item_t, NULL, NULL); /* @@ -506,7 +506,7 @@ typedef struct { /*--------------------------------------------------------------------*/ static void ibcm_component_register(void); -static int ibcm_component_query(mca_btl_openib_module_t *btl, +static int ibcm_component_query(mca_btl_openib_module_t *btl, ompi_btl_openib_connect_base_module_t **cpc); static int ibcm_component_finalize(void); @@ -544,13 +544,13 @@ ompi_btl_openib_connect_base_component_t ompi_btl_openib_connect_ibcm = { #define ENABLE_TIMERS (OPAL_ENABLE_DEBUG && 0) -#if ENABLE_TIMERS +#if ENABLE_TIMERS #include MCA_timer_IMPLEMENTATION_HEADER enum { QUERY, START_CONNECT, - QP_TO_RTR, + QP_TO_RTR, QP_TO_RTS, REQUEST_RECEIVED, REPLY_RECEIVED, @@ -616,7 +616,7 @@ static void ibcm_component_register(void) mca_base_param_reg_int(&mca_btl_openib_component.super.btl_version, "connect_ibcm_gid_index", "GID table index to use to obtain each port's GUID", - false, false, ibcm_gid_table_index, + false, false, ibcm_gid_table_index, &ibcm_gid_table_index); if (ibcm_gid_table_index < 0) { ibcm_gid_table_index = 0; @@ -625,15 +625,15 @@ static void ibcm_component_register(void) /*--------------------------------------------------------------------*/ -/* The IB_CM_ASSIGN_SERVICE_ID value passed to ib_cm_listen function asks, +/* The IB_CM_ASSIGN_SERVICE_ID value passed to ib_cm_listen function asks, * from IBCM , to assign service_id. - * The value was taken from IBCM kernel level + * The value was taken from IBCM kernel level */ #ifndef IB_CM_ASSIGN_SERVICE_ID #define IB_CM_ASSIGN_SERVICE_ID hton64(0x0200000000000000ULL) #endif -static int ibcm_component_query(mca_btl_openib_module_t *btl, +static int ibcm_component_query(mca_btl_openib_module_t *btl, ompi_btl_openib_connect_base_module_t **cpc) { int rc; @@ -760,11 +760,11 @@ static int ibcm_component_query(mca_btl_openib_module_t *btl, goto error; } OPAL_OUTPUT((-1, "opened ibcm device 0x%" PRIx64 " (%s:%d)", - (uint64_t) cmh->cm_device, + (uint64_t) cmh->cm_device, ibv_get_device_name(cmh->ib_context->device), openib_btl->port_num)); - if (0 != (rc = ib_cm_create_id(cmh->cm_device, + if (0 != (rc = ib_cm_create_id(cmh->cm_device, &cmh->listen_cm_id, NULL))) { /* Same rationale as above */ OBJ_RELEASE(cmh); @@ -774,12 +774,12 @@ static int ibcm_component_query(mca_btl_openib_module_t *btl, } if (0 != (rc = ib_cm_listen(cmh->listen_cm_id, IB_CM_ASSIGN_SERVICE_ID, 0))) { - /* Same rationale as above */ - OBJ_RELEASE(cmh); - BTL_ERROR(("failed to ib_cm_listen : rc=%d, errno=%d", rc, errno)); - rc = OMPI_ERR_NOT_SUPPORTED; - goto error; - } + /* Same rationale as above */ + OBJ_RELEASE(cmh); + BTL_ERROR(("failed to ib_cm_listen : rc=%d, errno=%d", rc, errno)); + rc = OMPI_ERR_NOT_SUPPORTED; + goto error; + } if (0 != (rc = ib_cm_attr_id(cmh->listen_cm_id, &(cmh->param)))) { OBJ_RELEASE(cmh); @@ -822,7 +822,7 @@ static int ibcm_component_query(mca_btl_openib_module_t *btl, rc = OMPI_ERR_UNREACH; goto error; } - rc = ibv_query_gid(btl->device->ib_dev_context, btl->port_num, ibcm_gid_table_index, + rc = ibv_query_gid(btl->device->ib_dev_context, btl->port_num, ibcm_gid_table_index, &gid); if (0 != rc) { BTL_ERROR(("system error (ibv_query_gid failed)")); @@ -863,8 +863,8 @@ static int ibcm_component_query(mca_btl_openib_module_t *btl, btl->port_num)); } else { BTL_VERBOSE(("unavailable for use on %s:%d; fatal error %d (%s)", - ibv_get_device_name(btl->device->ib_dev), - btl->port_num, rc, + ibv_get_device_name(btl->device->ib_dev), + btl->port_num, rc, opal_strerror(rc))); } return rc; @@ -895,8 +895,8 @@ static uint32_t max_inline_size(int qp, mca_btl_openib_device_t *device) * Create the local side of one qp. The remote side will be connected * later. */ -static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, - struct ibv_srq *srq, uint32_t max_recv_wr, +static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, + struct ibv_srq *srq, uint32_t max_recv_wr, uint32_t max_send_wr) { mca_btl_openib_module_t *openib_btl = endpoint->endpoint_btl; @@ -910,7 +910,7 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, init_attr.send_cq = openib_btl->device->ib_cq[BTL_OPENIB_LP_CQ]; init_attr.recv_cq = openib_btl->device->ib_cq[qp_cq_prio(qp)]; init_attr.srq = srq; - init_attr.cap.max_inline_data = req_inline = + init_attr.cap.max_inline_data = req_inline = max_inline_size(qp, openib_btl->device); init_attr.cap.max_send_sge = 1; init_attr.cap.max_recv_sge = 1; /* we do not use SG list */ @@ -922,10 +922,10 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, } init_attr.cap.max_send_wr = max_send_wr; - my_qp = ibv_create_qp(openib_btl->device->ib_pd, &init_attr); - if (NULL == my_qp) { - BTL_ERROR(("error creating qp errno says %s", strerror(errno))); - return OMPI_ERROR; + my_qp = ibv_create_qp(openib_btl->device->ib_pd, &init_attr); + if (NULL == my_qp) { + BTL_ERROR(("error creating qp errno says %s", strerror(errno))); + return OMPI_ERROR; } endpoint->qps[qp].qp->lcl_qp = my_qp; if (init_attr.cap.max_inline_data < req_inline) { @@ -971,7 +971,7 @@ static int qp_create_all(mca_btl_base_endpoint_t* endpoint, pp_qp_num = 1; } - for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) { + for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) { struct ibv_srq *srq = NULL; uint32_t max_recv_wr, max_send_wr; int32_t rd_rsv, rd_num_credits; @@ -985,7 +985,7 @@ static int qp_create_all(mca_btl_base_endpoint_t* endpoint, } if (BTL_OPENIB_QP_TYPE_PP(qp)) { - max_recv_wr = mca_btl_openib_component.qp_infos[qp].rd_num + + max_recv_wr = mca_btl_openib_component.qp_infos[qp].rd_num + rd_rsv; max_send_wr = mca_btl_openib_component.qp_infos[qp].rd_num + rd_num_credits; @@ -1019,14 +1019,14 @@ static int fill_path_record(ibcm_module_t *m, mca_btl_base_endpoint_t *endpoint, struct ibv_sa_path_rec *path_rec) { - modex_msg_t *remote_msg = + modex_msg_t *remote_msg = (modex_msg_t*) endpoint->endpoint_remote_cpc_data->cbm_modex_message; - modex_msg_t *local_msg = + modex_msg_t *local_msg = (modex_msg_t*) m->cpc.data.cbm_modex_message; /* Global attributes */ - path_rec->dgid.global.subnet_prefix = - path_rec->sgid.global.subnet_prefix = + path_rec->dgid.global.subnet_prefix = + path_rec->sgid.global.subnet_prefix = hton64(m->btl->port_info.subnet_id); path_rec->dgid.global.interface_id = hton64(remote_msg->mm_port_guid); path_rec->sgid.global.interface_id = hton64(local_msg->mm_port_guid); @@ -1064,7 +1064,7 @@ static int fill_path_record(ibcm_module_t *m, path_rec->pkey = mca_btl_openib_component.ib_pkey_val; if (0 == path_rec->pkey) { uint16_t pkey; - ibv_query_pkey(endpoint->endpoint_btl->device->ib_dev_context, + ibv_query_pkey(endpoint->endpoint_btl->device->ib_dev_context, endpoint->endpoint_btl->port_num, 0, &pkey); path_rec->pkey = ntohs(pkey); } @@ -1103,10 +1103,10 @@ static int fill_path_record(ibcm_module_t *m, BTL_VERBOSE(("Got src/dest subnet id: 0x%" PRIx64 " / 0x%" PRIx64, path_rec->sgid.global.subnet_prefix, path_rec->dgid.global.subnet_prefix)); - BTL_VERBOSE(("Got src/dest interface id: 0x%" PRIx64 " / 0x%" PRIx64, + BTL_VERBOSE(("Got src/dest interface id: 0x%" PRIx64 " / 0x%" PRIx64, path_rec->sgid.global.interface_id, path_rec->dgid.global.interface_id)); - BTL_VERBOSE(("Got src/dest lid: 0x%x / 0x%x", + BTL_VERBOSE(("Got src/dest lid: 0x%x / 0x%x", path_rec->slid, path_rec->dlid)); BTL_VERBOSE(("Got raw_traffic: %d", path_rec->raw_traffic)); @@ -1131,7 +1131,7 @@ static int fill_path_record(ibcm_module_t *m, static int ibcm_endpoint_init(struct mca_btl_base_endpoint_t *endpoint) { - ibcm_endpoint_t *ie = endpoint->endpoint_local_cpc_data = + ibcm_endpoint_t *ie = endpoint->endpoint_local_cpc_data = calloc(1, sizeof(ibcm_endpoint_t)); if (NULL == ie) { BTL_ERROR(("malloc failed!")); @@ -1141,7 +1141,7 @@ static int ibcm_endpoint_init(struct mca_btl_base_endpoint_t *endpoint) BTL_VERBOSE(("endpoint %p / %p", (void*)endpoint, (void*)ie)); ie->ie_cpc = endpoint->endpoint_local_cpc; ie->ie_endpoint = endpoint; - ie->ie_qps_created = + ie->ie_qps_created = ie->ie_recv_buffers_posted = false; ie->ie_qps_to_connect = mca_btl_openib_component.num_qps; @@ -1159,12 +1159,12 @@ static int ibcm_endpoint_init(struct mca_btl_base_endpoint_t *endpoint) static bool i_initiate(ibcm_module_t *m, mca_btl_openib_endpoint_t *endpoint) { - modex_msg_t *msg = + modex_msg_t *msg = (modex_msg_t*) endpoint->endpoint_remote_cpc_data->cbm_modex_message; - uint64_t my_port_guid = ntoh64(m->btl->device->ib_dev_attr.node_guid) + + uint64_t my_port_guid = ntoh64(m->btl->device->ib_dev_attr.node_guid) + m->btl->port_num; uint64_t service_id = m->cmh->param.service_id; - + BTL_VERBOSE(("i_initiate: my guid (%0" PRIx64 "), msg guid (%0" PRIx64 ")", my_port_guid, msg->mm_port_guid)); BTL_VERBOSE(("i_initiate: my service id (%d), msg service id (%d)", @@ -1172,7 +1172,7 @@ static bool i_initiate(ibcm_module_t *m, return (my_port_guid == msg->mm_port_guid && - service_id < msg->mm_service_id) ? true : + service_id < msg->mm_service_id) ? true : (my_port_guid < msg->mm_port_guid) ? true : false; } @@ -1187,11 +1187,11 @@ static ibcm_request_t *alloc_request(ibcm_module_t *m, modex_msg_t *msg, struct ib_cm_req_param *cm_req; ibcm_request_t *req = OBJ_NEW(ibcm_request_t); BTL_VERBOSE(("allocated cached req id: 0x%" PRIx64, (void*)req)); - + if (NULL == req) { return NULL; } - + /* Create this CM ID */ if (0 != ib_cm_create_id(m->cmh->cm_device, &(req->super.cm_id), @@ -1201,11 +1201,11 @@ static ibcm_request_t *alloc_request(ibcm_module_t *m, modex_msg_t *msg, return NULL; } BTL_VERBOSE(("created CM ID 0x%" PRIx64, &(req->super.cm_id))); - + /* This data is constant for all the QP's */ req->path_rec = *path_rec; req->endpoint = endpoint; - + cm_req = &(req->cm_req); cm_req->qp_type = IBV_QPT_RC; cm_req->alternate_path = NULL; @@ -1220,7 +1220,7 @@ static ibcm_request_t *alloc_request(ibcm_module_t *m, modex_msg_t *msg, cm_req->remote_cm_response_timeout = 20; cm_req->local_cm_response_timeout = 20; cm_req->max_cm_retries = 5; - + req->private_data.ireqd_pid = m->cmh->param.service_id; req->private_data.ireqd_ep_index = endpoint->index; @@ -1249,21 +1249,21 @@ static void print_req(struct ib_cm_req_param *cm_req) BTL_VERBOSE(("cm_req->max_cm_retries: %d", cm_req->max_cm_retries)); BTL_VERBOSE(("cm_req->srq: %d", cm_req->srq)); } - + static int ibcm_module_start_connect(ompi_btl_openib_connect_base_module_t *cpc, mca_btl_base_endpoint_t *endpoint) { int i, rc; ibcm_module_t *m = (ibcm_module_t *) cpc; - ibcm_endpoint_t *ie = + ibcm_endpoint_t *ie = (ibcm_endpoint_t *) endpoint->endpoint_local_cpc_data; - modex_msg_t *msg = + modex_msg_t *msg = (modex_msg_t*) endpoint->endpoint_remote_cpc_data->cbm_modex_message; struct ibv_sa_path_rec path_rec; bool do_initiate; TIMER_START(START_CONNECT); - BTL_VERBOSE(("endpoint %p (lid %d, ep index %d)", + BTL_VERBOSE(("endpoint %p (lid %d, ep index %d)", (void*)endpoint, endpoint->endpoint_btl->port_info.lid, endpoint->index)); @@ -1291,7 +1291,7 @@ static int ibcm_module_start_connect(ompi_btl_openib_connect_base_module_t *cpc, rc = OMPI_ERR_NOT_FOUND; goto err; } - + /* If we're not the initiator, make a bogus QP (must be done before we make all the other QPs) */ @@ -1303,14 +1303,14 @@ static int ibcm_module_start_connect(ompi_btl_openib_connect_base_module_t *cpc, } ie->ie_bogus_qp = endpoint->qps[0].qp->lcl_qp; } - + /* Make the local side of all the QP's */ if (OMPI_SUCCESS != (rc = qp_create_all(endpoint, m))) { goto err; } /* Check initiation direction (see comment above i_initiate() - function): + function): - if this is the side that is not supposed to initiate, then send a single bogus request that we expect to be rejected. @@ -1365,12 +1365,12 @@ static int ibcm_module_start_connect(ompi_btl_openib_connect_base_module_t *cpc, cm_req->srq = BTL_OPENIB_QP_TYPE_SRQ(i); cm_req->qp_num = endpoint->qps[i].qp->lcl_qp->qp_num; cm_req->starting_psn = endpoint->qps[i].qp->lcl_psn; - BTL_VERBOSE(("sending my qpn %d, psn %d", + BTL_VERBOSE(("sending my qpn %d, psn %d", cm_req->qp_num, cm_req->starting_psn)); - + req->private_data.ireqd_request = req; req->private_data.ireqd_qp_index = i; - + /* Send the request */ BTL_VERBOSE(("sending connect request %d of %d (id %p)", i, mca_btl_openib_component.num_qps, @@ -1409,7 +1409,7 @@ static int ibcm_module_start_connect(ompi_btl_openib_connect_base_module_t *cpc, cm_req->srq = 0; cm_req->qp_num = ie->ie_bogus_qp->qp_num; cm_req->starting_psn = 0; - BTL_VERBOSE(("sending BOGUS qpn %d, psn %d (id %p)", + BTL_VERBOSE(("sending BOGUS qpn %d, psn %d (id %p)", cm_req->qp_num, cm_req->starting_psn, (void*)req->super.cm_id)); @@ -1472,7 +1472,7 @@ static void ibcm_listen_cm_id_destructor(ibcm_listen_cm_id_t *cmh) /* Remove all the ibcm module items */ for (item = opal_list_remove_first(&(cmh->ibcm_modules)); - NULL != item; + NULL != item; item = opal_list_remove_first(&(cmh->ibcm_modules))) { OBJ_RELEASE(item); } @@ -1494,7 +1494,7 @@ static void ibcm_listen_cm_id_destructor(ibcm_listen_cm_id_t *cmh) /* Stop monitoring the cm_device's fd (wait for it to be released from the monitoring entity) */ - ompi_btl_openib_fd_unmonitor(cmh->cm_device->fd, + ompi_btl_openib_fd_unmonitor(cmh->cm_device->fd, callback_unlock, (void*) &barrier); @@ -1527,7 +1527,7 @@ static void ibcm_listen_cm_id_destructor(ibcm_listen_cm_id_t *cmh) /* Close the CM device */ if (NULL != cmh->cm_device) { OPAL_OUTPUT((-1, "closing ibcm device 0x%" PRIx64 " (%s)", - (uint64_t) cmh->cm_device, + (uint64_t) cmh->cm_device, ibv_get_device_name(cmh->ib_context->device))); ib_cm_close_device(cmh->cm_device); } @@ -1565,7 +1565,7 @@ static int ibcm_endpoint_finalize(struct mca_btl_base_endpoint_t *endpoint) ibcm_endpoint_t *ie = (ibcm_endpoint_t *) endpoint->endpoint_local_cpc_data; BTL_VERBOSE(("endpoint %p", (void*)endpoint)); - + /* Free the stuff we allocated in ibcm_module_init */ if (NULL != ie) { int i; @@ -1603,7 +1603,7 @@ static int ibcm_module_finalize(mca_btl_openib_module_t *btl, if (NULL != m && NULL != m->cmh) { OBJ_RELEASE(m->cmh); } - + return OMPI_SUCCESS; } @@ -1655,7 +1655,7 @@ static int qp_to_rtr(int qp_index, struct ib_cm_id *cm_id, (mtu == IBV_MTU_2048) ? "2048" : (mtu == IBV_MTU_4096) ? "4096" : "unknown (!)")); - + /* Move the QP into the INIT state */ memset(&attr, 0, sizeof(attr)); attr.qp_state = IBV_QPS_INIT; @@ -1665,9 +1665,9 @@ static int qp_to_rtr(int qp_index, struct ib_cm_id *cm_id, } if (0 != ibv_modify_qp(qp, &attr, attr_mask)) { - BTL_ERROR(("error modifying qp to INIT errno says %s", strerror(errno))); + BTL_ERROR(("error modifying qp to INIT errno says %s", strerror(errno))); return OMPI_ERROR; - } + } /* Move the QP into the RTR state */ attr.qp_state = IBV_QPS_RTR; @@ -1687,7 +1687,7 @@ static int qp_to_rtr(int qp_index, struct ib_cm_id *cm_id, /* IBM CM does not set these values for us */ attr.max_dest_rd_atomic = mca_btl_openib_component.ib_max_rdma_dst_ops; attr.min_rnr_timer = mca_btl_openib_component.ib_min_rnr_timer; - + if (0 != ibv_modify_qp(qp, &attr, attr_mask | IBV_QP_PATH_MTU | @@ -1696,9 +1696,9 @@ static int qp_to_rtr(int qp_index, struct ib_cm_id *cm_id, )) { BTL_ERROR(("error modifing QP to RTR errno says %s", strerror(errno))); - return OMPI_ERROR; + return OMPI_ERROR; } - + /* All done */ TIMER_STOP(QP_TO_RTR); return OMPI_SUCCESS; @@ -1727,9 +1727,9 @@ static int qp_to_rts(int qp_index, struct ib_cm_id *cm_id, if (0 != (rc = ibv_modify_qp(qp, &attr, attr_mask))) { BTL_ERROR(("error modifing QP (index %d) to RTS errno says %s; rc=%d, errno=%d", qp_index, strerror(errno), rc, errno)); - return OMPI_ERROR; + return OMPI_ERROR; } - + /* All done */ BTL_VERBOSE(("successfully set RTS")); TIMER_STOP(QP_TO_RTS); @@ -1742,7 +1742,7 @@ static int qp_to_rts(int qp_index, struct ib_cm_id *cm_id, */ static void *callback_start_connect(void *context) { - callback_start_connect_data_t *cbdata = + callback_start_connect_data_t *cbdata = (callback_start_connect_data_t *) context; BTL_VERBOSE(("ibcm scheduled callback: calling start_connect()")); @@ -1763,7 +1763,7 @@ static void *callback_start_connect(void *context) /* * Passive has received a connection request from a active */ -static int request_received(ibcm_listen_cm_id_t *cmh, +static int request_received(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) { int i, rc = OMPI_ERROR; @@ -1794,7 +1794,7 @@ static int request_received(ibcm_listen_cm_id_t *cmh, device event, and have to find the ibcm_module_t (i.e., local port/openib BTL module ) that corresponds to it. */ BTL_VERBOSE(("looking for ibcm module -- source port guid: 0x%" PRIx64 " (%p)", - ntoh64(req->primary_path->sgid.global.interface_id), + ntoh64(req->primary_path->sgid.global.interface_id), (void*)cmh)); for (item = opal_list_get_first(&(cmh->ibcm_modules)); item != opal_list_get_end(&(cmh->ibcm_modules)); @@ -1820,7 +1820,7 @@ static int request_received(ibcm_listen_cm_id_t *cmh, OPAL_THREAD_LOCK(&mca_btl_openib_component.ib_lock); for (found = false, ib_proc = (mca_btl_openib_proc_t*) opal_list_get_first(&mca_btl_openib_component.ib_procs); - !found && + !found && ib_proc != (mca_btl_openib_proc_t*) opal_list_get_end(&mca_btl_openib_component.ib_procs); ib_proc = (mca_btl_openib_proc_t*)opal_list_get_next(ib_proc)) { @@ -1828,7 +1828,7 @@ static int request_received(ibcm_listen_cm_id_t *cmh, /* Now cycle through all the endpoints on that proc */ for (i = 0; !found && i < (int) ib_proc->proc_endpoint_count; ++i) { BTL_VERBOSE(("checking endpoint %d of %d (ep %p, cpc data %p)", - i, (int) ib_proc->proc_endpoint_count, + i, (int) ib_proc->proc_endpoint_count, (void*)ib_proc->proc_endpoints[i], (void*)ib_proc->proc_endpoints[i]->endpoint_remote_cpc_data)); if (NULL == ib_proc->proc_endpoints[i]->endpoint_remote_cpc_data) { @@ -1841,7 +1841,7 @@ static int request_received(ibcm_listen_cm_id_t *cmh, BTL_VERBOSE(("my LID %d, remote LID %d", msg->mm_lid, ntohs(req->primary_path->dlid))); - if (msg->mm_port_guid == + if (msg->mm_port_guid == ntoh64(req->primary_path->dgid.global.interface_id) && msg->mm_service_id == active_private_data->ireqd_pid && msg->mm_port_num == req->port && @@ -1885,7 +1885,7 @@ static int request_received(ibcm_listen_cm_id_t *cmh, rc = OMPI_SUCCESS; } else if (ie->ie_connection_flags & CFLAGS_ONGOING) { /* See if the request for this QP already arrived */ - if (ie->ie_qps_created && + if (ie->ie_qps_created && IBV_QPS_RESET != endpoint->qps[qp_index].qp->lcl_qp->state) { BTL_VERBOSE(("this QP (%d) already connected", qp_index)); @@ -1934,7 +1934,7 @@ static int request_received(ibcm_listen_cm_id_t *cmh, are filled in during qp_to_rtr (because we don't get them until we call ib_cm_attr_init()). We already have the remote LID, subnet ID, and MTU from the port's modex message. */ - endpoint->rem_info.rem_qps[qp_index].rem_psn = + endpoint->rem_info.rem_qps[qp_index].rem_psn = event->param.req_rcvd.starting_psn; endpoint->rem_info.rem_index = active_private_data->ireqd_ep_index; @@ -2030,14 +2030,14 @@ static int request_received(ibcm_listen_cm_id_t *cmh, rep->cm_rep.qp_num = endpoint->qps[qp_index].qp->lcl_qp->qp_num; rep->cm_rep.srq = BTL_OPENIB_QP_TYPE_SRQ(qp_index); rep->cm_rep.starting_psn = endpoint->qps[qp_index].qp->lcl_psn; - BTL_VERBOSE(("setting reply psn %d", + BTL_VERBOSE(("setting reply psn %d", rep->cm_rep.starting_psn)); rep->cm_rep.responder_resources = req->responder_resources; rep->cm_rep.initiator_depth = req->initiator_depth; rep->cm_rep.target_ack_delay = 20; rep->cm_rep.flow_control = req->flow_control; rep->cm_rep.rnr_retry_count = req->rnr_retry_count; - + rep->private_data.irepd_request = active_private_data->ireqd_request; rep->private_data.irepd_reply = rep; rep->private_data.irepd_qp_index = qp_index; @@ -2050,7 +2050,7 @@ static int request_received(ibcm_listen_cm_id_t *cmh, goto reject; } opal_list_append(&ibcm_pending_replies, &(rep->super.super)); - + TIMER_STOP(REQUEST_RECEIVED); BTL_VERBOSE(("sent reply for qp index %d", qp_index)); return OMPI_SUCCESS; @@ -2058,21 +2058,21 @@ static int request_received(ibcm_listen_cm_id_t *cmh, reject: /* Reject the request */ BTL_VERBOSE(("rejecting request")); - ib_cm_send_rej(event->cm_id, IB_CM_REJ_CONSUMER_DEFINED, + ib_cm_send_rej(event->cm_id, IB_CM_REJ_CONSUMER_DEFINED, &rej_reason, sizeof(rej_reason), event->private_data, sizeof(ibcm_req_data_t)); - + /* If we rejected because of the wrong direction, then initiate a connection going the other direction. */ if (REJ_WRONG_DIRECTION == rej_reason) { callback_start_connect_data_t *cbdata = malloc(sizeof(*cbdata)); if (NULL != cbdata) { - cbdata->cscd_cpc = + cbdata->cscd_cpc = (ompi_btl_openib_connect_base_module_t *) imodule; cbdata->cscd_endpoint = endpoint; BTL_VERBOSE(("starting connect in other direction")); ompi_btl_openib_fd_run_in_main(callback_start_connect, cbdata); - + TIMER_STOP(REQUEST_RECEIVED); return OMPI_SUCCESS; } @@ -2085,7 +2085,7 @@ static int request_received(ibcm_listen_cm_id_t *cmh, endpoint); return rc; } - + /* * Callback (from main thread) when the endpoint has been connected */ @@ -2102,8 +2102,8 @@ static void *callback_set_endpoint_cpc_complete(void *context) /* * Helper function to find a cached CM ID in a list - */ -static ibcm_base_cm_id_t *find_cm_id(struct ib_cm_id *cm_id, + */ +static ibcm_base_cm_id_t *find_cm_id(struct ib_cm_id *cm_id, opal_list_t *list) { opal_list_item_t *item; @@ -2139,7 +2139,7 @@ static int reply_received(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) p->irepd_qp_index, (void*) endpoint)); ie = (ibcm_endpoint_t*) endpoint->endpoint_local_cpc_data; - endpoint->rem_info.rem_qps[p->irepd_qp_index].rem_psn = + endpoint->rem_info.rem_qps[p->irepd_qp_index].rem_psn = event->param.rep_rcvd.starting_psn; endpoint->rem_info.rem_index = p->irepd_ep_index; @@ -2253,7 +2253,7 @@ static int ready_to_use_received(ibcm_listen_cm_id_t *h, static int reject_received(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) { enum ib_cm_rej_reason reason = event->param.rej_rcvd.reason; - ibcm_reject_reason_t *rej_reason = + ibcm_reject_reason_t *rej_reason = (ibcm_reject_reason_t *) event->param.rej_rcvd.ari; TIMER_START(REJECT_RECEIVED); @@ -2268,7 +2268,7 @@ static int reject_received(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) (ibcm_req_data_t*) event->private_data; ibcm_request_t *request = my_private_data->ireqd_request; mca_btl_openib_endpoint_t *endpoint = request->endpoint; - ibcm_endpoint_t *ie = (ibcm_endpoint_t*) + ibcm_endpoint_t *ie = (ibcm_endpoint_t*) endpoint->endpoint_local_cpc_data; BTL_VERBOSE(("got WRONG_DIRECTION reject, endpoint: %p, pid %d, ep_index %d, qp_index %d", @@ -2283,9 +2283,9 @@ static int reject_received(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) /* Remove from the global pending_requests list because we no longer need to handle errors for it */ BTL_VERBOSE(("reply received cm id %p -- original cached req %p", - (void*)cmh->listen_cm_id, + (void*)cmh->listen_cm_id, (void*)request)); - opal_list_remove_item(&ibcm_pending_requests, + opal_list_remove_item(&ibcm_pending_requests, &(request->super.super)); /* We ack the event and then destroy the CM ID (you *must* @@ -2326,13 +2326,13 @@ static int request_error(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) if (IBV_WC_RESP_TIMEOUT_ERR != event->param.send_status) { orte_show_help("help-mpi-btl-openib-cpc-ibcm.txt", "unhandled error", true, - "request", orte_process_info.nodename, + "request", orte_process_info.nodename, event->param.send_status); } else { ibcm_request_t *req; - BTL_ERROR(("Got timeout in IBCM request (CM ID: %p)", + BTL_ERROR(("Got timeout in IBCM request (CM ID: %p)", (void*)event->cm_id)); - req = (ibcm_request_t*) find_cm_id(event->cm_id, + req = (ibcm_request_t*) find_cm_id(event->cm_id, &ibcm_pending_requests); if (NULL == req) { orte_show_help("help-mpi-btl-openib-cpc-ibcm.txt", @@ -2345,7 +2345,7 @@ static int request_error(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) /* Communicate to the upper layer that the connection on this endpoint has failed */ - ompi_btl_openib_fd_run_in_main(mca_btl_openib_endpoint_invoke_error, + ompi_btl_openib_fd_run_in_main(mca_btl_openib_endpoint_invoke_error, endpoint); return OMPI_SUCCESS; } @@ -2358,13 +2358,13 @@ static int reply_error(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) if (IBV_WC_RESP_TIMEOUT_ERR != event->param.send_status) { orte_show_help("help-mpi-btl-openib-cpc-ibcm.txt", "unhandled error", true, - "reply", orte_process_info.nodename, + "reply", orte_process_info.nodename, event->param.send_status); } else { ibcm_reply_t *rep; BTL_ERROR(("Got timeout in IBCM reply (id: %p)", (void*)event->cm_id)); - rep = (ibcm_reply_t*) find_cm_id(event->cm_id, + rep = (ibcm_reply_t*) find_cm_id(event->cm_id, &ibcm_pending_replies); if (NULL == rep) { orte_show_help("help-mpi-btl-openib-cpc-ibcm.txt", @@ -2377,7 +2377,7 @@ static int reply_error(ibcm_listen_cm_id_t *cmh, struct ib_cm_event *event) /* Communicate to the upper layer that the connection on this endpoint has failed */ - ompi_btl_openib_fd_run_in_main(mca_btl_openib_endpoint_invoke_error, + ompi_btl_openib_fd_run_in_main(mca_btl_openib_endpoint_invoke_error, endpoint); return OMPI_SUCCESS; } @@ -2390,7 +2390,7 @@ static void *ibcm_event_dispatch(int fd, int flags, void *context) ibcm_listen_cm_id_t *cmh = (ibcm_listen_cm_id_t*) context; struct ib_cm_event *e = NULL; - OPAL_OUTPUT((-1, "ibcm dispatch: on device 0x%" PRIx64", fd %d", + OPAL_OUTPUT((-1, "ibcm dispatch: on device 0x%" PRIx64", fd %d", (uint64_t) cmh->cm_device, fd)); TIMER_START(CM_GET_EVENT); /* Blocks until next event, which should be immediately (because @@ -2415,19 +2415,19 @@ static void *ibcm_event_dispatch(int fd, int flags, void *context) /* Incoming request */ rc = request_received(cmh, e); break; - + case IB_CM_REP_RECEIVED: OPAL_OUTPUT((-1, "ibcm dispatch: reply received on fd %d", fd)); /* Reply received */ rc = reply_received(cmh, e); break; - + case IB_CM_RTU_RECEIVED: OPAL_OUTPUT((-1, "ibcm dispatch: RTU received on fd %d", fd)); /* Ready to use! */ rc = ready_to_use_received(cmh, e); break; - + case IB_CM_REJ_RECEIVED: OPAL_OUTPUT((-1, "ibcm dispatch: reject received on fd %d", fd)); /* Rejected connection */ @@ -2436,19 +2436,19 @@ static void *ibcm_event_dispatch(int fd, int flags, void *context) ID could be freed */ want_ack = false; break; - + case IB_CM_REQ_ERROR: OPAL_OUTPUT((-1, "ibcm dispatch: request error received on fd %d", fd)); /* Request error */ rc = request_error(cmh, e); break; - + case IB_CM_REP_ERROR: OPAL_OUTPUT((-1, "ibcm dispatch: reply error received on fd %d", fd)); /* Reply error */ rc = reply_error(cmh, e); break; - + case IB_CM_DREQ_RECEIVED: case IB_CM_DREP_RECEIVED: case IB_CM_DREQ_ERROR: @@ -2459,7 +2459,7 @@ static void *ibcm_event_dispatch(int fd, int flags, void *context) /* We don't care */ rc = OMPI_SUCCESS; break; - + default: /* This would be odd */ OPAL_OUTPUT((-1, "ibcm dispatch: unhandled event received on fd %d", fd)); diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.h b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.h index 2d834b6641..36a8734fee 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.h +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_ibcm.h @@ -2,9 +2,9 @@ * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c b/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c index e6e1d49b11..211737bbd0 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_oob.c @@ -5,21 +5,21 @@ * Copyright (c) 2004-2011 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006 Los Alamos National Security, LLC. All rights - * reserved. + * reserved. * Copyright (c) 2008-2009 Mellanox Technologies. All rights reserved. * Copyright (c) 2009 IBM Corporation. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -39,7 +39,7 @@ #include "ompi/mca/dpm/dpm.h" #include "btl_openib.h" -#include "btl_openib_endpoint.h" +#include "btl_openib_endpoint.h" #include "btl_openib_proc.h" #include "connect/connect.h" #include "orte/util/show_help.h" @@ -88,7 +88,7 @@ static int oob_priority = 50; static bool rml_recv_posted = false; static void oob_component_register(void); -static int oob_component_query(mca_btl_openib_module_t *openib_btl, +static int oob_component_query(mca_btl_openib_module_t *openib_btl, ompi_btl_openib_connect_base_module_t **cpc); static int oob_component_finalize(void); @@ -102,14 +102,14 @@ static int qp_connect_all(mca_btl_base_endpoint_t* endpoint); static int qp_create_all(mca_btl_base_endpoint_t* endpoint); static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, struct ibv_srq *srq, uint32_t max_recv_wr, uint32_t max_send_wr); -static int send_connect_data(mca_btl_base_endpoint_t* endpoint, +static int send_connect_data(mca_btl_base_endpoint_t* endpoint, uint8_t message_type); -static void rml_send_cb(int status, orte_process_name_t* endpoint, - opal_buffer_t* buffer, orte_rml_tag_t tag, +static void rml_send_cb(int status, orte_process_name_t* endpoint, + opal_buffer_t* buffer, orte_rml_tag_t tag, void* cbdata); -static void rml_recv_cb(int status, orte_process_name_t* process_name, - opal_buffer_t* buffer, orte_rml_tag_t tag, +static void rml_recv_cb(int status, orte_process_name_t* process_name, + opal_buffer_t* buffer, orte_rml_tag_t tag, void* cbdata); #if (ENABLE_DYNAMIC_SL) @@ -172,7 +172,7 @@ static void oob_component_register(void) * Init function. Post non-blocking RML receive to accept incoming * connection requests. */ -static int oob_component_query(mca_btl_openib_module_t *btl, +static int oob_component_query(mca_btl_openib_module_t *btl, ompi_btl_openib_connect_base_module_t **cpc) { int rc; @@ -180,7 +180,7 @@ static int oob_component_query(mca_btl_openib_module_t *btl, /* If we have the transport_type member, check to ensure we're on IB (this CPC will not work with iWarp). If we do not have the transport_type member, then we must be < OFED v1.2, and - therefore we must be IB. */ + therefore we must be IB. */ #if defined(HAVE_STRUCT_IBV_DEVICE_TRANSPORT_TYPE) if (BTL_OPENIB_CONNECT_BASE_CHECK_IF_NOT_IB(btl)) { opal_output_verbose(5, mca_btl_base_output, @@ -202,7 +202,7 @@ static int oob_component_query(mca_btl_openib_module_t *btl, ensure to only post it *once*, because another btl may have come in before this and already posted it. */ if (!rml_recv_posted) { - rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, + rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, OMPI_RML_TAG_OPENIB, ORTE_RML_PERSISTENT, rml_recv_cb, @@ -261,7 +261,7 @@ static int oob_module_start_connect(ompi_btl_openib_connect_base_module_t *cpc, endpoint->endpoint_state = MCA_BTL_IB_CONNECTING; if (OMPI_SUCCESS != (rc = send_connect_data(endpoint, ENDPOINT_CONNECT_REQUEST))) { - BTL_ERROR(("error sending connect request, error code %d", rc)); + BTL_ERROR(("error sending connect request, error code %d", rc)); return rc; } @@ -330,7 +330,7 @@ static int reply_start_connect(mca_btl_openib_endpoint_t *endpoint, /* Set the remote side info */ set_remote_info(endpoint, rem_info); - + /* Connect to remote endpoint qp's */ if (OMPI_SUCCESS != (rc = qp_connect_all(endpoint))) { return rc; @@ -352,14 +352,14 @@ static int set_remote_info(mca_btl_base_endpoint_t* endpoint, mca_btl_openib_rem_info_t* rem_info) { /* copy the rem_info stuff */ - memcpy(&((mca_btl_openib_endpoint_t*) endpoint)->rem_info, - rem_info, sizeof(mca_btl_openib_rem_info_t)); - + memcpy(&((mca_btl_openib_endpoint_t*) endpoint)->rem_info, + rem_info, sizeof(mca_btl_openib_rem_info_t)); + /* copy over the rem qp info */ memcpy(endpoint->rem_info.rem_qps, - rem_info->rem_qps, sizeof(mca_btl_openib_rem_qp_info_t) * + rem_info->rem_qps, sizeof(mca_btl_openib_rem_qp_info_t) * mca_btl_openib_component.num_qps); - + BTL_VERBOSE(("Setting QP info, LID = %d", endpoint->rem_info.rem_lid)); return ORTE_SUCCESS; @@ -433,7 +433,7 @@ static int qp_connect_all(mca_btl_openib_endpoint_t *endpoint) IBV_QP_MIN_RNR_TIMER)) { BTL_ERROR(("error modifing QP to RTR errno says %s", strerror(errno))); - return OMPI_ERROR; + return OMPI_ERROR; } attr.qp_state = IBV_QPS_RTS; attr.timeout = mca_btl_openib_component.ib_timeout; @@ -482,7 +482,7 @@ static int qp_create_all(mca_btl_base_endpoint_t* endpoint) if(0 == pp_qp_num && true == endpoint->use_eager_rdma) pp_qp_num = 1; - for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) { + for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) { struct ibv_srq *srq = NULL; uint32_t max_recv_wr, max_send_wr; int32_t rd_rsv, rd_num_credits; @@ -540,7 +540,7 @@ static uint32_t max_inline_size(int qp, mca_btl_openib_device_t *device) * Create the local side of one qp. The remote side will be connected * later. */ -static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, +static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, struct ibv_srq *srq, uint32_t max_recv_wr, uint32_t max_send_wr) { mca_btl_openib_module_t *openib_btl = endpoint->endpoint_btl; @@ -556,7 +556,7 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, init_attr.send_cq = openib_btl->device->ib_cq[BTL_OPENIB_LP_CQ]; init_attr.recv_cq = openib_btl->device->ib_cq[qp_cq_prio(qp)]; init_attr.srq = srq; - init_attr.cap.max_inline_data = req_inline = + init_attr.cap.max_inline_data = req_inline = max_inline_size(qp, openib_btl->device); init_attr.cap.max_send_sge = 1; init_attr.cap.max_recv_sge = 1; /* we do not use SG list */ @@ -567,11 +567,11 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, } init_attr.cap.max_send_wr = max_send_wr; - my_qp = ibv_create_qp(openib_btl->device->ib_pd, &init_attr); - - if (NULL == my_qp) { - BTL_ERROR(("error creating qp errno says %s", strerror(errno))); - return OMPI_ERROR; + my_qp = ibv_create_qp(openib_btl->device->ib_pd, &init_attr); + + if (NULL == my_qp) { + BTL_ERROR(("error creating qp errno says %s", strerror(errno))); + return OMPI_ERROR; } endpoint->qps[qp].qp->lcl_qp = my_qp; @@ -585,21 +585,21 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, } else { endpoint->qps[qp].ib_inline_max = req_inline; } - + attr.qp_state = IBV_QPS_INIT; attr.pkey_index = openib_btl->pkey_index; attr.port_num = openib_btl->port_num; attr.qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; - if (ibv_modify_qp(endpoint->qps[qp].qp->lcl_qp, - &attr, - IBV_QP_STATE | - IBV_QP_PKEY_INDEX | - IBV_QP_PORT | - IBV_QP_ACCESS_FLAGS )) { - BTL_ERROR(("error modifying qp to INIT errno says %s", strerror(errno))); - return OMPI_ERROR; - } + if (ibv_modify_qp(endpoint->qps[qp].qp->lcl_qp, + &attr, + IBV_QP_STATE | + IBV_QP_PKEY_INDEX | + IBV_QP_PORT | + IBV_QP_ACCESS_FLAGS )) { + BTL_ERROR(("error modifying qp to INIT errno says %s", strerror(errno))); + return OMPI_ERROR; + } /* Setup meta data on the endpoint */ endpoint->qps[qp].qp->lcl_psn = lrand48() & 0xffffff; @@ -612,18 +612,18 @@ static int qp_create_one(mca_btl_base_endpoint_t* endpoint, int qp, /* * RML send connect information to remote endpoint */ -static int send_connect_data(mca_btl_base_endpoint_t* endpoint, +static int send_connect_data(mca_btl_base_endpoint_t* endpoint, uint8_t message_type) { opal_buffer_t* buffer = OBJ_NEW(opal_buffer_t); int rc; - + if (NULL == buffer) { ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE); return ORTE_ERR_OUT_OF_RESOURCE; } - /* pack the info in the send buffer */ + /* pack the info in the send buffer */ BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT8)); rc = opal_dss.pack(buffer, &message_type, 1, OPAL_UINT8); if (ORTE_SUCCESS != rc) { @@ -659,7 +659,7 @@ static int send_connect_data(mca_btl_base_endpoint_t* endpoint, if (message_type != ENDPOINT_CONNECT_ACK) { int qp; /* stuff all the QP info into the buffer */ - for (qp = 0; qp < mca_btl_openib_component.num_qps; qp++) { + for (qp = 0; qp < mca_btl_openib_component.num_qps; qp++) { BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32)); rc = opal_dss.pack(buffer, &endpoint->qps[qp].qp->lcl_qp->qp_num, 1, OPAL_UINT32); @@ -669,13 +669,13 @@ static int send_connect_data(mca_btl_base_endpoint_t* endpoint, } BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT32)); rc = opal_dss.pack(buffer, &endpoint->qps[qp].qp->lcl_psn, 1, - OPAL_UINT32); + OPAL_UINT32); if (ORTE_SUCCESS != rc) { ORTE_ERROR_LOG(rc); return rc; } } - + BTL_VERBOSE(("packing %d of %d\n", 1, OPAL_UINT16)); rc = opal_dss.pack(buffer, &endpoint->endpoint_btl->lid, 1, OPAL_UINT16); if (ORTE_SUCCESS != rc) { @@ -698,7 +698,7 @@ static int send_connect_data(mca_btl_base_endpoint_t* endpoint, } /* send to remote endpoint */ - rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_ompi->proc_name, + rc = orte_rml.send_buffer_nb(&endpoint->endpoint_proc->proc_ompi->proc_name, buffer, OMPI_RML_TAG_OPENIB, 0, rml_send_cb, NULL); if (ORTE_SUCCESS != rc) { @@ -706,7 +706,7 @@ static int send_connect_data(mca_btl_base_endpoint_t* endpoint, return rc; } BTL_VERBOSE(("Sent QP Info, LID = %d, SUBNET = %" PRIx64 "\n", - endpoint->endpoint_btl->lid, + endpoint->endpoint_btl->lid, endpoint->subnet_id)); return OMPI_SUCCESS; @@ -717,8 +717,8 @@ static int send_connect_data(mca_btl_base_endpoint_t* endpoint, * Callback when we have finished RML sending the connect data to a * remote peer */ -static void rml_send_cb(int status, orte_process_name_t* endpoint, - opal_buffer_t* buffer, orte_rml_tag_t tag, +static void rml_send_cb(int status, orte_process_name_t* endpoint, + opal_buffer_t* buffer, orte_rml_tag_t tag, void* cbdata) { OBJ_RELEASE(buffer); @@ -730,8 +730,8 @@ static void rml_send_cb(int status, orte_process_name_t* endpoint, * and if this endpoint is trying to connect, reply with our QP info, * otherwise try to modify QP's and establish reliable connection */ -static void rml_recv_cb(int status, orte_process_name_t* process_name, - opal_buffer_t* buffer, orte_rml_tag_t tag, +static void rml_recv_cb(int status, orte_process_name_t* process_name, + opal_buffer_t* buffer, orte_rml_tag_t tag, void* cbdata) { mca_btl_openib_proc_t *ib_proc; @@ -744,9 +744,9 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, mca_btl_openib_rem_info_t rem_info; uint8_t message_type; bool master; - - /* start by unpacking data first so we know who is knocking at - our door */ + + /* start by unpacking data first so we know who is knocking at + our door */ BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT8)); rc = opal_dss.unpack(buffer, &message_type, &cnt, OPAL_UINT8); if (ORTE_SUCCESS != rc) { @@ -754,7 +754,7 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, mca_btl_openib_endpoint_invoke_error(NULL); return; } - + BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT64)); rc = opal_dss.unpack(buffer, &rem_info.rem_subnet_id, &cnt, OPAL_UINT64); if (ORTE_SUCCESS != rc) { @@ -762,7 +762,7 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, mca_btl_openib_endpoint_invoke_error(NULL); return; } - + if (ENDPOINT_CONNECT_REQUEST != message_type) { BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT32)); rc = opal_dss.unpack(buffer, &lcl_qp, &cnt, OPAL_UINT32); @@ -780,14 +780,14 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, } } if (ENDPOINT_CONNECT_ACK != message_type) { - int qp; + int qp; /* get ready for the data */ - rem_info.rem_qps = - (mca_btl_openib_rem_qp_info_t*) malloc(sizeof(mca_btl_openib_rem_qp_info_t) * + rem_info.rem_qps = + (mca_btl_openib_rem_qp_info_t*) malloc(sizeof(mca_btl_openib_rem_qp_info_t) * mca_btl_openib_component.num_qps); - + /* unpack all the qp info */ - for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) { + for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) { BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT32)); rc = opal_dss.unpack(buffer, &rem_info.rem_qps[qp].rem_qp_num, &cnt, OPAL_UINT32); @@ -805,7 +805,7 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, return; } } - + BTL_VERBOSE(("unpacking %d of %d\n", cnt, OPAL_UINT16)); rc = opal_dss.unpack(buffer, &rem_info.rem_lid, &cnt, OPAL_UINT16); if (ORTE_SUCCESS != rc) { @@ -828,14 +828,14 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, return; } } - + BTL_VERBOSE(("Received QP Info, LID = %d, SUBNET = %" PRIx64 "\n", - rem_info.rem_lid, + rem_info.rem_lid, rem_info.rem_subnet_id)); - + master = orte_util_compare_name_fields(ORTE_NS_CMP_ALL, ORTE_PROC_MY_NAME, process_name) > 0 ? true : false; - + /* Need to protect the ib_procs list */ OPAL_THREAD_LOCK(&mca_btl_openib_component.ib_lock); @@ -845,16 +845,16 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, opal_list_get_end(&mca_btl_openib_component.ib_procs); ib_proc = (mca_btl_openib_proc_t*)opal_list_get_next(ib_proc)) { bool found = false; - + if (orte_util_compare_name_fields(ORTE_NS_CMP_ALL, &ib_proc->proc_ompi->proc_name, process_name) != OPAL_EQUAL) { continue; } - + if (ENDPOINT_CONNECT_REQUEST != message_type) { /* This is a reply message. Try to get the endpoint instance the reply belongs to */ - for (i = 0; i < ib_proc->proc_endpoint_count; i++) { + for (i = 0; i < ib_proc->proc_endpoint_count; i++) { ib_endpoint = ib_proc->proc_endpoints[i]; if (ib_endpoint->qps[0].qp->lcl_qp != NULL && lcl_lid == ib_endpoint->endpoint_btl->lid && @@ -872,7 +872,7 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, mca_btl_openib_endpoint_t *ib_endpoint_found = NULL; int master_first_closed = -1; - for (i = 0; i < ib_proc->proc_endpoint_count; i++) { + for (i = 0; i < ib_proc->proc_endpoint_count; i++) { ib_endpoint = ib_proc->proc_endpoints[i]; if (ib_endpoint->subnet_id != rem_info.rem_subnet_id || (ib_endpoint->endpoint_state != MCA_BTL_IB_CONNECTING @@ -894,7 +894,7 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, break; /* Found one. No point to continue */ } ib_endpoint = ib_endpoint_found; - + if (found && master && MCA_BTL_IB_CLOSED == ib_endpoint->endpoint_state ) { /* since this is master and no endpoints found in @@ -912,17 +912,17 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, return; } } - + if (!found) { - BTL_ERROR(("can't find suitable endpoint for this peer\n")); + BTL_ERROR(("can't find suitable endpoint for this peer\n")); mca_btl_openib_endpoint_invoke_error(NULL); OPAL_THREAD_UNLOCK(&mca_btl_openib_component.ib_lock); - return; + return; } - + OPAL_THREAD_LOCK(&ib_endpoint->endpoint_lock); endpoint_state = ib_endpoint->endpoint_state; - + /* Update status */ switch (endpoint_state) { case MCA_BTL_IB_CLOSED : @@ -933,17 +933,17 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, if (master) { rc = reply_start_connect(ib_endpoint, &rem_info); } else { - rc = oob_module_start_connect(ib_endpoint->endpoint_local_cpc, + rc = oob_module_start_connect(ib_endpoint->endpoint_local_cpc, ib_endpoint); } - + if (OMPI_SUCCESS != rc) { BTL_ERROR(("error in endpoint reply start connect")); mca_btl_openib_endpoint_invoke_error(ib_endpoint); OPAL_THREAD_UNLOCK(&ib_endpoint->endpoint_lock); break; } - + /* As long as we expect a message from the peer (in order to setup the connection) let the event engine pool the RML events. Note: we increment it once peer active @@ -951,16 +951,16 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, opal_progress_event_users_increment(); OPAL_THREAD_UNLOCK(&ib_endpoint->endpoint_lock); break; - + case MCA_BTL_IB_CONNECTING : set_remote_info(ib_endpoint, &rem_info); if (OMPI_SUCCESS != (rc = qp_connect_all(ib_endpoint))) { - BTL_ERROR(("endpoint connect error: %d", rc)); + BTL_ERROR(("endpoint connect error: %d", rc)); mca_btl_openib_endpoint_invoke_error(ib_endpoint); OPAL_THREAD_UNLOCK(&ib_endpoint->endpoint_lock); break; } - + if (master) { ib_endpoint->endpoint_state = MCA_BTL_IB_WAITING_ACK; @@ -974,20 +974,20 @@ static void rml_recv_cb(int status, orte_process_name_t* process_name, /* cpc complete unlock the endpoint */ } break; - + case MCA_BTL_IB_WAITING_ACK: /* Tell main BTL that we're done */ mca_btl_openib_endpoint_cpc_complete(ib_endpoint); /* cpc complete unlock the endpoint */ break; - + case MCA_BTL_IB_CONNECT_ACK: send_connect_data(ib_endpoint, ENDPOINT_CONNECT_ACK); /* Tell main BTL that we're done */ mca_btl_openib_endpoint_cpc_complete(ib_endpoint); /* cpc complete unlock the endpoint */ break; - + case MCA_BTL_IB_CONNECTED: OPAL_THREAD_UNLOCK(&ib_endpoint->endpoint_lock); break; diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_oob.h b/ompi/mca/btl/openib/connect/btl_openib_connect_oob.h index 1adf0c21ff..21c483cf97 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_oob.h +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_oob.h @@ -2,9 +2,9 @@ * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c b/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c index 24c72a6141..178a70aea9 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c @@ -108,7 +108,7 @@ typedef struct { static void rdmacm_contents_constructor(rdmacm_contents_t *contents); static void rdmacm_contents_destructor(rdmacm_contents_t *contents); -OBJ_CLASS_INSTANCE(rdmacm_contents_t, opal_list_item_t, +OBJ_CLASS_INSTANCE(rdmacm_contents_t, opal_list_item_t, rdmacm_contents_constructor, rdmacm_contents_destructor); @@ -142,7 +142,7 @@ typedef struct { static void id_context_constructor(id_context_t *context); static void id_context_destructor(id_context_t *context); -OBJ_CLASS_INSTANCE(id_context_t, opal_list_item_t, +OBJ_CLASS_INSTANCE(id_context_t, opal_list_item_t, id_context_constructor, id_context_destructor); @@ -225,7 +225,7 @@ static void rdmacm_contents_destructor(rdmacm_contents_t *contents) /* * Invoked by main thread * - * Sets up any rdma_cm specific commandline params + * Sets up any rdma_cm specific commandline params */ static void rdmacm_component_register(void) { @@ -288,7 +288,7 @@ static void rdmacm_component_register(void) static char *stringify(uint32_t addr) { char *line = (char *) malloc(64); - asprintf(&line, "%d.%d.%d.%d (0x%x)", + asprintf(&line, "%d.%d.%d.%d (0x%x)", #if defined(WORDS_BIGENDIAN) (addr >> 24), (addr >> 16) & 0xff, @@ -306,7 +306,7 @@ static char *stringify(uint32_t addr) /* * Invoked by service thread - * + * * This function traverses the list of endpoints associated with the * device and determines which of them the remote side is attempting * to connect to. This is determined based on the local endpoint's @@ -360,7 +360,7 @@ static mca_btl_openib_endpoint_t *rdmacm_find_endpoint(rdmacm_contents_t *conten } /* - * Returns max inlne size for qp #N + * Returns max inlne size for qp #N */ static uint32_t max_inline_size(int qp, mca_btl_openib_device_t *device) { @@ -425,7 +425,7 @@ static int rdmacm_setup_qp(rdmacm_contents_t *contents, attr.cap.max_recv_wr = 0; } attr.cap.max_send_wr = max_send_wr; - attr.cap.max_inline_data = req_inline = + attr.cap.max_inline_data = req_inline = max_inline_size(qpnum, contents->openib_btl->device); attr.cap.max_send_sge = 1; attr.cap.max_recv_sge = 1; /* we do not use SG list */ @@ -473,7 +473,7 @@ out: } -/* +/* * Invoked by both main and service threads * * To avoid all kinds of nasty race conditions, we only allow @@ -490,7 +490,7 @@ static bool i_initiate(uint32_t local_ipaddr, uint16_t local_port, char *a = stringify(local_ipaddr); char *b = stringify(remote_ipaddr); #endif - + if (local_ipaddr > remote_ipaddr || (local_ipaddr == remote_ipaddr && local_port < remote_port)) { OPAL_OUTPUT((-1, "i_initiate (I WIN): local ipaddr %s, remote ipaddr %s", @@ -564,10 +564,10 @@ static int rdmacm_client_connect_one(rdmacm_contents_t *contents, * RDMA_CM_EVENT_ADDR_RESOLVED event will occur on the local event * handler. */ - OPAL_OUTPUT((-1, "MAIN Resolving id: from IP %s:%d to IP %s:%d", - a = stringify(contents->ipaddr), + OPAL_OUTPUT((-1, "MAIN Resolving id: from IP %s:%d to IP %s:%d", + a = stringify(contents->ipaddr), contents->tcp_port, - b = stringify(message->ipaddr), + b = stringify(message->ipaddr), message->tcp_port)); #if OPAL_ENABLE_DEBUG free(a); @@ -629,7 +629,7 @@ out: return OMPI_ERROR; } -/* +/* * Invoked by main thread * * Connect method called by the upper layers to connect the local @@ -651,7 +651,7 @@ static int rdmacm_module_start_connect(ompi_btl_openib_connect_base_module_t *cp invoked from the event_handler (to intitiate connections in the Right direction), where we don't have the CPC, so it'll be NULL. */ - local_message = + local_message = (modex_message_t *) endpoint->endpoint_local_cpc->data.cbm_modex_message; message = (modex_message_t *) endpoint->endpoint_remote_cpc_data->cbm_modex_message; @@ -698,8 +698,8 @@ static int rdmacm_module_start_connect(ompi_btl_openib_connect_base_module_t *cp /* Are we the initiator? Or do we expect this connect request to be rejected? */ - endpoint->endpoint_initiator = - i_initiate(contents->ipaddr, contents->tcp_port, + endpoint->endpoint_initiator = + i_initiate(contents->ipaddr, contents->tcp_port, message->ipaddr, message->tcp_port); OPAL_OUTPUT((-1, "MAIN Start connect; ep=%p (%p), I %s the initiator to %s", (void*) endpoint, @@ -711,7 +711,7 @@ static int rdmacm_module_start_connect(ompi_btl_openib_connect_base_module_t *cp if (contents->endpoint->endpoint_initiator) { /* Initiator needs a CTS frag (non-initiator will have a CTS frag allocated later) */ - if (OMPI_SUCCESS != + if (OMPI_SUCCESS != (rc = ompi_btl_openib_connect_base_alloc_cts(contents->endpoint))) { BTL_ERROR(("Failed to alloc CTS frag")); goto out; @@ -720,7 +720,7 @@ static int rdmacm_module_start_connect(ompi_btl_openib_connect_base_module_t *cp for (qp = 0; qp < mca_btl_openib_component.num_qps; qp++) { rc = rdmacm_client_connect_one(contents, message, qp); if (OMPI_SUCCESS != rc) { - BTL_ERROR(("rdmacm_client_connect_one error (real QP %d)", + BTL_ERROR(("rdmacm_client_connect_one error (real QP %d)", qp)); goto out; } @@ -750,7 +750,7 @@ out: static void *show_help_cant_find_endpoint(void *context) { char *msg; - cant_find_endpoint_context_t *c = + cant_find_endpoint_context_t *c = (cant_find_endpoint_context_t*) context; if (NULL != c) { @@ -774,7 +774,7 @@ static void *show_help_cant_find_endpoint(void *context) return NULL; } -/* +/* * Invoked by service thread * * The server thread will handle the incoming connection requests and @@ -822,12 +822,12 @@ static int handle_connect_request(struct rdma_cm_event *event) } message = (modex_message_t *) endpoint->endpoint_remote_cpc_data->cbm_modex_message; - endpoint->endpoint_initiator = + endpoint->endpoint_initiator = i_initiate(contents->ipaddr, contents->tcp_port, message->ipaddr, rem_port); BTL_VERBOSE(("ep state = %d, local ipaddr = %x, remote ipaddr = %x, local port = %d, remote port = %d", - endpoint->endpoint_state, contents->ipaddr, message->ipaddr, + endpoint->endpoint_state, contents->ipaddr, message->ipaddr, contents->tcp_port, rem_port)); OPAL_OUTPUT((-1, "SERVICE in handle_connect_request; ep=%p (%p), I still %s the initiator to %s", @@ -879,7 +879,7 @@ static int handle_connect_request(struct rdma_cm_event *event) if (mca_btl_openib_component.credits_qp == qpnum) { struct ibv_recv_wr *bad_wr, *wr; - if (OMPI_SUCCESS != + if (OMPI_SUCCESS != ompi_btl_openib_connect_base_alloc_cts(endpoint)) { BTL_ERROR(("Failed to alloc CTS frag")); goto out1; @@ -888,7 +888,7 @@ static int handle_connect_request(struct rdma_cm_event *event) assert(NULL != wr); wr->next = NULL; - if (0 != ibv_post_recv(endpoint->qps[qpnum].qp->lcl_qp, + if (0 != ibv_post_recv(endpoint->qps[qpnum].qp->lcl_qp, wr, &bad_wr)) { BTL_ERROR(("failed to post CTS recv buffer")); goto out1; @@ -923,10 +923,10 @@ static int handle_connect_request(struct rdma_cm_event *event) /* See rdma_connect(3) for a description of these 2 values. We ensure to pass these values around via the modex so that we can compute the values properly. */ - conn_param.responder_resources = + conn_param.responder_resources = mymin(contents->openib_btl->device->ib_dev_attr.max_qp_rd_atom, message->device_max_qp_init_rd_atom); - conn_param.initiator_depth = + conn_param.initiator_depth = mymin(contents->openib_btl->device->ib_dev_attr.max_qp_init_rd_atom, message->device_max_qp_rd_atom); conn_param.retry_count = mca_btl_openib_component.ib_retry_count; @@ -1001,9 +1001,9 @@ static void *call_disconnect_callback(void *v) return NULL; } -/* +/* * Invoked by main thread - * + * * Runs *while* the progress thread is running. We can't stop the * progress thread because this function may be invoked to kill a * specific endpoint that was the result of MPI-2 dynamics (i.e., this @@ -1033,12 +1033,12 @@ static int rdmacm_endpoint_finalize(struct mca_btl_base_endpoint_t *endpoint) opal_mutex_lock(&client_list_lock); num_to_wait_for = disconnect_callbacks = 0; for (item = opal_list_get_first(&client_list); - item != opal_list_get_end(&client_list); + item != opal_list_get_end(&client_list); item = opal_list_get_next(item)) { rdmacm_contents_t *contents = (rdmacm_contents_t *) item; if (endpoint == contents->endpoint) { - while (NULL != + while (NULL != (item2 = opal_list_remove_first(&(contents->ids)))) { /* Fun race condition: we cannot call rdma_disconnect() here in the main thread, because @@ -1131,7 +1131,7 @@ static int rdmacm_connect_endpoint(id_context_t *context, BTL_ERROR(("Can't find endpoint")); return OMPI_ERR_NOT_FOUND; } - data = + data = (rdmacm_endpoint_local_cpc_data_t *)endpoint->endpoint_local_cpc_data; /* Only notify the upper layers after the last QP has been @@ -1210,7 +1210,7 @@ static int rdmacm_rejected(id_context_t *context, struct rdma_cm_event *event) /* Why were we rejected? */ switch (*((reject_reason_t*) event->param.conn.private_data)) { case REJECT_WRONG_DIRECTION: - OPAL_OUTPUT((-1, "SERVICE A good reject! for qp %d, id 0x%p", + OPAL_OUTPUT((-1, "SERVICE A good reject! for qp %d, id 0x%p", context->qpnum, (void*) context->id)); rdmacm_destroy_dummy_qp(context); break; @@ -1261,10 +1261,10 @@ out: /* * Runs in service thread */ -static int create_dummy_cq(rdmacm_contents_t *contents, +static int create_dummy_cq(rdmacm_contents_t *contents, mca_btl_openib_module_t *openib_btl) { - contents->dummy_cq = + contents->dummy_cq = ibv_create_cq(openib_btl->device->ib_dev_context, 1, NULL, NULL, 0); if (NULL == contents->dummy_cq) { BTL_ERROR(("dummy_cq not created")); @@ -1279,7 +1279,7 @@ out: /* * Runs in service thread */ -static int create_dummy_qp(rdmacm_contents_t *contents, +static int create_dummy_qp(rdmacm_contents_t *contents, struct rdma_cm_id *id, int qpnum) { struct ibv_qp_init_attr attr; @@ -1347,7 +1347,7 @@ static int finish_connect(id_context_t *context) /* If we're the initiator, then setup the QP's and post the CTS message buffer */ if (contents->endpoint->endpoint_initiator) { - rc = rdmacm_setup_qp(contents, contents->endpoint, + rc = rdmacm_setup_qp(contents, contents->endpoint, context->id, context->qpnum); if (0 != rc) { BTL_ERROR(("rdmacm_setup_qp error %d", rc)); @@ -1357,14 +1357,14 @@ static int finish_connect(id_context_t *context) if (mca_btl_openib_component.credits_qp == context->qpnum) { /* Post a single receive buffer on the smallest QP for the CTS protocol */ - + struct ibv_recv_wr *bad_wr, *wr; assert(NULL != contents->endpoint->endpoint_cts_frag.super.super.base.super.ptr); wr = &(contents->endpoint->endpoint_cts_frag.rd_desc); assert(NULL != wr); wr->next = NULL; - - if (0 != ibv_post_recv(contents->endpoint->qps[context->qpnum].qp->lcl_qp, + + if (0 != ibv_post_recv(contents->endpoint->qps[context->qpnum].qp->lcl_qp, wr, &bad_wr)) { BTL_ERROR(("failed to post CTS recv buffer")); goto out1; @@ -1399,10 +1399,10 @@ static int finish_connect(id_context_t *context) memset(&conn_param, 0, sizeof(conn_param)); /* See above comment about rdma_connect(3) and these two values. */ - conn_param.responder_resources = + conn_param.responder_resources = mymin(contents->openib_btl->device->ib_dev_attr.max_qp_rd_atom, message->device_max_qp_init_rd_atom); - conn_param.initiator_depth = + conn_param.initiator_depth = mymin(contents->openib_btl->device->ib_dev_attr.max_qp_init_rd_atom, message->device_max_qp_rd_atom); conn_param.flow_control = 0; @@ -1456,7 +1456,7 @@ out: return OMPI_ERROR; } -/* +/* * Runs in main thread */ static void *show_help_rdmacm_event_error(void *c) @@ -1566,14 +1566,14 @@ static int event_handler(struct rdma_cm_event *event) found = false; if (OMPI_SUCCESS == ompi_btl_openib_ini_query(attr->vendor_id, attr->vendor_part_id, - &ini) && + &ini) && ini.rdmacm_reject_causes_connect_error) { found = true; } if (rdmacm_reject_causes_connect_error) { found = true; } - + if (found) { OPAL_OUTPUT((-1, "SERVICE Got CONNECT_ERROR, but ignored: %p", (void*) event->id)); rc = rdmacm_destroy_dummy_qp(context); @@ -1628,7 +1628,7 @@ static inline void rdmamcm_event_error(struct rdma_cm_event *event) endpoint = ((id_context_t *)event->id->context)->contents->endpoint; } - ompi_btl_openib_fd_run_in_main(mca_btl_openib_endpoint_invoke_error, + ompi_btl_openib_fd_run_in_main(mca_btl_openib_endpoint_invoke_error, endpoint); } @@ -1684,7 +1684,7 @@ static void *rdmacm_event_dispatch(int fd, int flags, void *context) /* * Runs in main thread * - * CPC init function - Setup all globals here + * CPC init function - Setup all globals here */ static int rdmacm_init(mca_btl_openib_endpoint_t *endpoint) { @@ -1716,14 +1716,14 @@ static int ipaddrcheck(id_context_t *context, * up). Unfortunately, the subnet and IP address look up needs to match or * there could be a mismatch if IP Aliases are being used. For more * information on this, please read comment above - * mca_btl_openib_get_ip_subnet_id in btl_openib_ip.c + * mca_btl_openib_get_ip_subnet_id in btl_openib_ip.c */ - ipaddr = - mca_btl_openib_rdma_get_ipv4addr(openib_btl->device->ib_dev_context, + ipaddr = + mca_btl_openib_rdma_get_ipv4addr(openib_btl->device->ib_dev_context, openib_btl->port_num); if (0 == ipaddr) { BTL_VERBOSE(("*** Could not find IP address for %s:%d -- is there an IP address configured for this device?", - ibv_get_device_name(openib_btl->device->ib_dev), + ibv_get_device_name(openib_btl->device->ib_dev), openib_btl->port_num)); return OMPI_ERR_NOT_FOUND; } @@ -1735,16 +1735,16 @@ static int ipaddrcheck(id_context_t *context, /* Ok, we found the IP address of this device/port. Have we already see this IP address/TCP port before? */ - for (item = opal_list_get_first(&server_listener_list); - item != opal_list_get_end(&server_listener_list); + for (item = opal_list_get_first(&server_listener_list); + item != opal_list_get_end(&server_listener_list); item = opal_list_get_next(item)) { rdmacm_contents_t *contents = (rdmacm_contents_t *)item; - BTL_VERBOSE(("paddr = %x, ipaddr addr = %x", + BTL_VERBOSE(("paddr = %x, ipaddr addr = %x", contents->ipaddr, ipaddr)); if (contents->ipaddr == ipaddr && contents->tcp_port == server_tcp_port) { str = stringify(ipaddr); - BTL_VERBOSE(("server already listening on %s:%d", + BTL_VERBOSE(("server already listening on %s:%d", str, server_tcp_port)); free(str); already_exists = true; @@ -1755,7 +1755,7 @@ static int ipaddrcheck(id_context_t *context, /* If we haven't seen it before, save it */ if (!already_exists) { str = stringify(ipaddr); - BTL_VERBOSE(("creating new server to listen on %s:%d", + BTL_VERBOSE(("creating new server to listen on %s:%d", str, server_tcp_port)); free(str); server->ipaddr = ipaddr; @@ -1765,8 +1765,8 @@ static int ipaddrcheck(id_context_t *context, return already_exists ? OMPI_ERROR : OMPI_SUCCESS; } -static int create_message(rdmacm_contents_t *server, - mca_btl_openib_module_t *openib_btl, +static int create_message(rdmacm_contents_t *server, + mca_btl_openib_module_t *openib_btl, ompi_btl_openib_connect_base_module_data_t *data) { modex_message_t *message; @@ -1780,14 +1780,14 @@ static int create_message(rdmacm_contents_t *server, return OMPI_ERR_OUT_OF_RESOURCE; } - message->device_max_qp_rd_atom = + message->device_max_qp_rd_atom = openib_btl->device->ib_dev_attr.max_qp_rd_atom; - message->device_max_qp_init_rd_atom = + message->device_max_qp_init_rd_atom = openib_btl->device->ib_dev_attr.max_qp_init_rd_atom; message->ipaddr = server->ipaddr; message->tcp_port = server->tcp_port; - OPAL_OUTPUT((-1, "Message IP address is %s, port %d", + OPAL_OUTPUT((-1, "Message IP address is %s, port %d", a = stringify(message->ipaddr), message->tcp_port)); #if OPAL_ENABLE_DEBUG free(a); @@ -1798,7 +1798,7 @@ static int create_message(rdmacm_contents_t *server, return OMPI_SUCCESS; } -/* +/* * Runs in main thread * * This function determines if the RDMACM is a possible cpc method and @@ -1943,7 +1943,7 @@ out: } else { opal_output_verbose(5, mca_btl_base_output, "openib BTL: rmacm CPC unavailable for use on %s:%d; fatal error %d (%s)", - ibv_get_device_name(openib_btl->device->ib_dev), + ibv_get_device_name(openib_btl->device->ib_dev), openib_btl->port_num, rc, opal_strerror(rc)); } @@ -1971,7 +1971,7 @@ static int rdmacm_component_finalize(void) if (NULL != event_channel) { #ifndef __WINDOWS__ - rc = ompi_btl_openib_fd_unmonitor(event_channel->fd, + rc = ompi_btl_openib_fd_unmonitor(event_channel->fd, rdmacm_unmonitor, (void*) &barrier); #endif if (OMPI_SUCCESS != rc) { @@ -2048,7 +2048,7 @@ static int rdmacm_component_init(void) ompi_btl_openib_fd_monitor(event_channel->fd, OPAL_EV_READ, rdmacm_event_dispatch, NULL); #endif - + rdmacm_component_initialized = true; return OMPI_SUCCESS; } diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.h b/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.h index df48502694..8a52dc5a9d 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.h +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.h @@ -2,9 +2,9 @@ * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c b/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c index 551f75d955..aabd04d38c 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.c @@ -36,7 +36,7 @@ #include "orte/util/show_help.h" static void xoob_component_register(void); -static int xoob_component_query(mca_btl_openib_module_t *openib_btl, +static int xoob_component_query(mca_btl_openib_module_t *openib_btl, ompi_btl_openib_connect_base_module_t **cpc); static int xoob_component_finalize(void); @@ -812,7 +812,7 @@ static void xoob_rml_recv_cb(int status, orte_process_name_t* process_name, requested_lid, message_type); if ( NULL == ib_endpoint) { BTL_ERROR(("Got ENDPOINT_XOOB_CONNECT_REQUEST." - " Failed to find endpoint with subnet %" PRIx64 + " Failed to find endpoint with subnet %" PRIx64 " and LID %d", rem_info.rem_subnet_id,requested_lid)); mca_btl_openib_endpoint_invoke_error(NULL); @@ -899,7 +899,7 @@ static void xoob_rml_recv_cb(int status, orte_process_name_t* process_name, /* update ib_addr with remote qp number */ ib_endpoint->ib_addr->remote_xrc_rcv_qp_num = ib_endpoint->rem_info.rem_qps->rem_qp_num; - BTL_VERBOSE(("rem_info: lid %d, sid %" PRIx64 + BTL_VERBOSE(("rem_info: lid %d, sid %" PRIx64 " ep %d %" PRIx64 "\n", rem_info.rem_lid, rem_info.rem_subnet_id, @@ -962,7 +962,7 @@ static void xoob_rml_recv_cb(int status, orte_process_name_t* process_name, */ /* Quere for the XOOB priority - will be highest in XRC case */ -static int xoob_component_query(mca_btl_openib_module_t *openib_btl, +static int xoob_component_query(mca_btl_openib_module_t *openib_btl, ompi_btl_openib_connect_base_module_t **cpc) { int rc; @@ -986,7 +986,7 @@ static int xoob_component_query(mca_btl_openib_module_t *openib_btl, ensure to only post it *once*, because another btl may have come in before this and already posted it. */ if (!rml_recv_posted) { - rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, + rc = orte_rml.recv_buffer_nb(ORTE_NAME_WILDCARD, OMPI_RML_TAG_XOPENIB, ORTE_RML_PERSISTENT, xoob_rml_recv_cb, @@ -999,7 +999,7 @@ static int xoob_component_query(mca_btl_openib_module_t *openib_btl, } rml_recv_posted = true; } - + (*cpc)->data.cbm_component = &ompi_btl_openib_connect_xoob; (*cpc)->data.cbm_priority = xoob_priority; (*cpc)->data.cbm_modex_message = NULL; diff --git a/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.h b/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.h index 0909d7e861..bedbb0726f 100644 --- a/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.h +++ b/ompi/mca/btl/openib/connect/btl_openib_connect_xoob.h @@ -3,9 +3,9 @@ * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ diff --git a/ompi/mca/btl/openib/connect/connect.h b/ompi/mca/btl/openib/connect/connect.h index cade129b68..297e368d10 100644 --- a/ompi/mca/btl/openib/connect/connect.h +++ b/ompi/mca/btl/openib/connect/connect.h @@ -2,9 +2,9 @@ * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. * * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -30,26 +30,26 @@ * * - a BTL module represents a network port (in the case of the openib * BTL, a LID) - * - a CPC module represents one way to make connections to a BTL module + * - a CPC module represents one way to make connections to a BTL module * - hence, a BTL module has potentially multiple CPC modules * associated with it - * - an endpoint represnts a connection between a local BTL module and - * a remote BTL module (in the openib BTL, because of BSRQ, an - * endpoint can contain multiple QPs) + * - an endpoint represnts a connection between a local BTL module and + * a remote BTL module (in the openib BTL, because of BSRQ, an + * endpoint can contain multiple QPs) * - when an endpoint is created, one of the CPC modules associated - * with the local BTL is selected and associated with the endpoint + * with the local BTL is selected and associated with the endpoint * (obviously, it is a CPC module that is common between the local - * and remote BTL modules) - * - endpoints may be created and destroyed during the MPI job - * - endpoints are created lazily, during the first communication - * between two peers - * - endpoints are destroyed when two MPI processes become - * disconnected (e.g., MPI-2 dynamics or MPI_FINALIZE) - * - hence, BTL modules and CPC modules outlive endpoints. - * Specifically, BTL modules and CPC modules live from MPI_INIT to - * MPI_FINALIZE. endpoints come and go as MPI semantics demand it. - * - therefore, CPC modules need to cache information on endpoints that - * are specific to that connection. + * and remote BTL modules) + * - endpoints may be created and destroyed during the MPI job + * - endpoints are created lazily, during the first communication + * between two peers + * - endpoints are destroyed when two MPI processes become + * disconnected (e.g., MPI-2 dynamics or MPI_FINALIZE) + * - hence, BTL modules and CPC modules outlive endpoints. + * Specifically, BTL modules and CPC modules live from MPI_INIT to + * MPI_FINALIZE. endpoints come and go as MPI semantics demand it. + * - therefore, CPC modules need to cache information on endpoints that + * are specific to that connection. * * Component interface: * @@ -57,7 +57,7 @@ * calls the connect_base_register() function, which scans all * compiled-in CPC's. If they have component_register() functions, * they are called (component_register() functions are only allowed to - * register MCA parameters). + * register MCA parameters). * * NOTE: The connect_base_register() function will process the * btl_openib_cpc_include and btl_openib_cpc_exclude MCA parameters @@ -230,7 +230,7 @@ typedef int (*ompi_btl_openib_connect_base_component_init_fn_t)(void); * - Other OMPI_ERR_* code: an error occurred. */ typedef int (*ompi_btl_openib_connect_base_func_component_query_t) - (struct mca_btl_openib_module_t *btl, + (struct mca_btl_openib_module_t *btl, struct ompi_btl_openib_connect_base_module_t **cpc); /** diff --git a/ompi/mca/btl/openib/help-mpi-btl-openib.txt b/ompi/mca/btl/openib/help-mpi-btl-openib.txt index e26aabaea9..b8019f6cf1 100644 --- a/ompi/mca/btl/openib/help-mpi-btl-openib.txt +++ b/ompi/mca/btl/openib/help-mpi-btl-openib.txt @@ -6,7 +6,7 @@ # Copyright (c) 2004-2005 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2006 The Regents of the University of California. # All rights reserved. @@ -14,9 +14,9 @@ # Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved. # Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. # $COPYRIGHT$ -# +# # Additional copyrights may follow -# +# # $HEADER$ # # This is the US/English help file for Open MPI's OpenFabrics support @@ -151,9 +151,9 @@ sent. This error usually means one of two things: -1. There is something awry within the network fabric itself. +1. There is something awry within the network fabric itself. 2. A bug in Open MPI has caused flow control to malfunction. - + #1 is usually more likely. You should note the hosts on which this error has occurred; it has been observed that rebooting or removing a particular host from the job can sometimes resolve this issue. @@ -200,7 +200,7 @@ exceeded. "Retry count" is defined in the InfiniBand spec 1.2 This error typically means that there is something awry within the InfiniBand fabric itself. You should note the hosts on which this error has occurred; it has been observed that rebooting or removing a -particular host from the job can sometimes resolve this issue. +particular host from the job can sometimes resolve this issue. Two MCA parameters can be used to control Open MPI's behavior with respect to the retry count: @@ -280,7 +280,7 @@ Deactivating the OpenFabrics BTL. Wrong buffer alignment %d configured on host '%s'. Should be bigger than zero and power of two. Use default %d instead. # -[of error event] +[of error event] The OpenFabrics stack has reported a network error event. Open MPI will try to continue, but your job may end up failing. @@ -591,7 +591,7 @@ conflict: WARNING: The openib BTL was directed to use "eager RDMA" for short messages, but the openib BTL was compiled with progress threads support. Short eager RDMA is not yet supported with progress threads; -its use has been disabled in this job. +its use has been disabled in this job. This is a warning only; you job will attempt to continue. # @@ -644,7 +644,7 @@ be able to run successfully. Local host: %s Local adapter: %s (vendor 0x%x, part ID %d) Local queues: %s - + Remote host: %s Remote adapter: (vendor 0x%x, part ID %d) Remote queues: %s @@ -656,7 +656,7 @@ Such mixed network trasport configuration is not supported by Open MPI. Local host: %s Local adapter: %s (vendor 0x%x, part ID %d) Local transport type: %s - + Remote host: %s Remote Adapter: (vendor 0x%x, part ID %d) Remote transport type: %s diff --git a/ompi/mca/btl/openib/mca-btl-openib-device-params.ini b/ompi/mca/btl/openib/mca-btl-openib-device-params.ini index 15ce262243..b64a765c5d 100644 --- a/ompi/mca/btl/openib/mca-btl-openib-device-params.ini +++ b/ompi/mca/btl/openib/mca-btl-openib-device-params.ini @@ -2,7 +2,7 @@ # Copyright (c) 2006-2009 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2006-2008 Mellanox Technologies. All rights reserved. # $COPYRIGHT$ -# +# # Additional copyrights may follow #