From 3cc8b0461a1a3ccb491e3ca8689fb5abcde7e6c1 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Wed, 12 Sep 2012 20:47:47 +0000 Subject: [PATCH] More updates to common verbs infrastructure: * Moved "check basics" sanity check from openib BTL to common/verbs (which also allows us to have openib ''not'' include , which is a Very Good Thing) * Add new ompi_common_verbs_qp_test() function, which tests to see whether a device supports RC and/or UD QPs. The openib BTL now uses this function to ensure that the device supports RC QPs. * Rename ompi_common_verbs_find_ibv_ports() to be ompi_common_verbs_find_ports() -- the "ibv" was redundant. * Re-work ompi_common_verbs_find_ports() to use ompi_common_verbs_qp_test() instead of testing for RC/UD QPs itself * Add bunches of opal_output_verbose() to the find_ports() routine (to help diagnosing connectivity problems -- imaging running with --mca btl_base_verbose 10; you'll see all the find_ports() test results) * Make ompi_common_verbs_qp_test() warn if devices/ports are supplied in the if_include/if_exclude strings that do not exists (quite similar to what the openib BTL does today). * Add ompi_common_verbs_mca_register() function, which registers common verbs MCA params. It will also register MCA param synonyms for thse MCA params to upper-level components (e.g., btl__). * common_verbs_warn_nonexistent_if: warn if if_include/if_exclude-specified devices or ports do not exist. This commit was SVN r27332. --- ompi/mca/btl/openib/btl_openib_component.c | 55 ++-- ompi/mca/common/verbs/Makefile.am | 5 +- ompi/mca/common/verbs/common_verbs.h | 47 +++- ompi/mca/common/verbs/common_verbs_basics.c | 65 +++++ .../common/verbs/common_verbs_find_ports.c | 265 ++++++++++++------ ompi/mca/common/verbs/common_verbs_mca.c | 56 ++++ ompi/mca/common/verbs/common_verbs_qp_type.c | 99 +++++++ .../common/verbs/help-ompi-common-verbs.txt | 12 + 8 files changed, 470 insertions(+), 134 deletions(-) create mode 100644 ompi/mca/common/verbs/common_verbs_basics.c create mode 100644 ompi/mca/common/verbs/common_verbs_mca.c create mode 100644 ompi/mca/common/verbs/common_verbs_qp_type.c diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index bcc8cc3577..665f196e3f 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -28,17 +28,6 @@ #include "ompi_config.h" #include -/* This is crummy, but doesn't work on all - platforms with all compilers. Specifically, trying to include it - on RHEL4U3 with the PGI 32 bit compiler will cause problems because - certain 64 bit types are not defined. Per advice from Roland D., - just include the one prototype that we need in this case - (ibv_get_sysfs_path()). */ -#ifdef HAVE_INFINIBAND_DRIVER_H -#include -#else -const char *ibv_get_sysfs_path(void); -#endif #include #include #ifdef HAVE_UNISTD_H @@ -284,29 +273,6 @@ static int btl_openib_component_close(void) return rc; } -static bool check_basics(void) -{ - int rc; - char *file; - struct stat s; - -#if defined(__linux__) - /* Check to see if $sysfsdir/class/infiniband/ exists */ - asprintf(&file, "%s/class/infiniband", ibv_get_sysfs_path()); - if (NULL == file) { - return false; - } - rc = stat(file, &s); - free(file); - if (0 != rc || !S_ISDIR(s.st_mode)) { - return false; - } -#endif - - /* It exists and is a directory -- good enough */ - return true; -} - static void inline pack8(char **dest, uint8_t value) { /* Copy one character */ @@ -1628,10 +1594,27 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) ompi_btl_openib_ini_values_t values, default_values; int *allowed_ports = NULL; bool need_search; + struct ibv_context *dev_context = NULL; + + /* Open up the device */ + dev_context = ibv_open_device(ib_dev); + if (NULL == dev_context) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + + /* Find out if this device supports RC QPs */ + if (OMPI_SUCCESS != ompi_common_verbs_qp_test(dev_context, + OMPI_COMMON_VERBS_FLAGS_RC)) { + ibv_close_device(dev_context); + BTL_VERBOSE(("openib: RC QPs not supported -- skipping %s", + ibv_get_device_name(ib_dev))); + return OMPI_ERR_NOT_SUPPORTED; + } device = OBJ_NEW(mca_btl_openib_device_t); if(NULL == device){ BTL_ERROR(("Failed malloc: %s:%d", __FILE__, __LINE__)); + ibv_close_device(dev_context); return OMPI_ERR_OUT_OF_RESOURCE; } @@ -1640,7 +1623,7 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev) device->mem_reg_max = 1ull << 48; device->ib_dev = ib_dev; - device->ib_dev_context = ibv_open_device(ib_dev); + device->ib_dev_context = dev_context; device->ib_pd = NULL; device->device_btls = OBJ_NEW(opal_pointer_array_t); if (OPAL_SUCCESS != opal_pointer_array_init(device->device_btls, 2, INT_MAX, 2)) { @@ -2539,7 +2522,7 @@ btl_openib_component_init(int *num_btl_modules, assume that the RDMA hardware drivers are not loaded, and therefore we don't want OpenFabrics verbs support in this OMPI job. No need to print a warning. */ - if (!check_basics()) { + if (!ompi_common_verbs_check_basics()) { goto no_btls; } diff --git a/ompi/mca/common/verbs/Makefile.am b/ompi/mca/common/verbs/Makefile.am index 509a643e7a..7788d3a75b 100644 --- a/ompi/mca/common/verbs/Makefile.am +++ b/ompi/mca/common/verbs/Makefile.am @@ -15,9 +15,12 @@ headers = \ common_verbs.h sources = \ + common_verbs_basics.c \ common_verbs_devlist.c \ common_verbs_find_ports.c \ - common_verbs_port.c + common_verbs_mca.c \ + common_verbs_port.c \ + common_verbs_qp_type.c dist_pkgdata_DATA = \ help-ompi-common-verbs.txt diff --git a/ompi/mca/common/verbs/common_verbs.h b/ompi/mca/common/verbs/common_verbs.h index 413b09ca57..f6042cc298 100644 --- a/ompi/mca/common/verbs/common_verbs.h +++ b/ompi/mca/common/verbs/common_verbs.h @@ -15,6 +15,11 @@ #include "ompi_config.h" +#include +#include + +#include "opal/mca/mca.h" + #include #include "opal/class/opal_list.h" @@ -27,6 +32,17 @@ BEGIN_C_DECLS OMPI_DECLSPEC struct ibv_device **ompi_ibv_get_device_list(int *num_devs); OMPI_DECLSPEC void ompi_ibv_free_device_list(struct ibv_device **ib_devs); +/* + * common_verbs_mca.c + */ +extern bool ompi_common_verbs_warn_nonexistent_if; +OMPI_DECLSPEC void ompi_common_verbs_mca_register(mca_base_component_t *component); + +/* + * common_verbs_basics.c + */ +bool ompi_common_verbs_check_basics(void); + /* * common_verbs_find_ports.c */ @@ -63,16 +79,21 @@ enum { OMPI_COMMON_VERBS_FLAGS_TRANSPORT_IB = 0x4, OMPI_COMMON_VERBS_FLAGS_TRANSPORT_IWARP = 0x8, /* Note that these 2 link layer flags will only be useful if - OMPI_HAVE_IBV_LINK_LAYER is set to 1. Otherwise, they will be + defined(HAVE_IBV_LINK_LAYER_ETHERNET). Otherwise, they will be ignored. */ OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_IB = 0x10, OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET = 0x20, OMPI_COMMON_VERBS_FLAGS_MAX }; -/* +/** * Find a list of ibv_device ports that match a specific criteria. * + * @param if_include (IN): comma-delimited list of interfaces to use + * @param if_exclude (IN): comma-delimited list of interfaces to NOT use + * @param flags (IN): bit flags + * @param verbose_stream (IN): stream to send opal_output_verbose messages to + * * The ports will adhere to the if_include / if_exclude lists (only * one can be specified). The lists are comma-delimited tokens in one * of two forms: @@ -86,6 +107,9 @@ enum { * it includes any of the capabilities/characteristics listed in the * flags. * + * Note that if the verbose_stream is >=0, output will be sent to that + * stream with a verbose level of 5. + * * A valid list will always be returned. It will contain zero or more * ompi_common_verbs_port_item_t items. Each item can be individually * OBJ_RELEASE'd; the destructor will take care of cleaning up the @@ -93,9 +117,10 @@ enum { * port_items referring to it have been freed). */ OMPI_DECLSPEC opal_list_t * -ompi_common_verbs_find_ibv_ports(const char *if_include, - const char *if_exclude, - int flags); +ompi_common_verbs_find_ports(const char *if_include, + const char *if_exclude, + int flags, + int verbose_stream); /* * Trivial function to compute the bandwidth on an ibv_port. @@ -115,6 +140,18 @@ ompi_common_verbs_port_bw(struct ibv_port_attr *port_attr, OMPI_DECLSPEC int ompi_common_verbs_mtu(struct ibv_port_attr *port_attr); +/* + * Test a device to see if it can handle a specific QP type (RC and/or + * UD). Will return the logical AND if multiple types are specified + * (e.g., if (RC|UD) are in flags, then will return OMPI_SUCCESS only + * if *both* types can be created on the device). + * + * Flags can be the logical OR of OMPI_COMMON_VERBS_FLAGS_RC and/or + * OMPI_COMMON_VERBS_FLAGS_UD. All other values are ignored. + */ +OMPI_DECLSPEC int ompi_common_verbs_qp_test(struct ibv_context *device_context, + int flags); + END_C_DECLS #endif diff --git a/ompi/mca/common/verbs/common_verbs_basics.c b/ompi/mca/common/verbs/common_verbs_basics.c new file mode 100644 index 0000000000..8821718bee --- /dev/null +++ b/ompi/mca/common/verbs/common_verbs_basics.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif + +/* This is crummy, but doesn't work on all + platforms with all compilers. Specifically, trying to include it + on RHEL4U3 with the PGI 32 bit compiler will cause problems because + certain 64 bit types are not defined. Per advice from Roland D., + just include the one prototype that we need in this case + (ibv_get_sysfs_path()). */ +#include +#ifdef HAVE_INFINIBAND_DRIVER_H +#include +#else +const char *ibv_get_sysfs_path(void); +#endif + +#include "ompi/constants.h" + +#include "common_verbs.h" + +/***********************************************************************/ + +bool ompi_common_verbs_check_basics(void) +{ +#if defined(__linux__) + int rc; + char *file; + struct stat s; + + /* Check to see if $sysfsdir/class/infiniband/ exists */ + asprintf(&file, "%s/class/infiniband", ibv_get_sysfs_path()); + if (NULL == file) { + return false; + } + rc = stat(file, &s); + free(file); + if (0 != rc || !S_ISDIR(s.st_mode)) { + return false; + } +#endif + + /* It exists and is a directory -- good enough */ + return true; +} + diff --git a/ompi/mca/common/verbs/common_verbs_find_ports.c b/ompi/mca/common/verbs/common_verbs_find_ports.c index 0cd8bb30d5..52f70af451 100644 --- a/ompi/mca/common/verbs/common_verbs_find_ports.c +++ b/ompi/mca/common/verbs/common_verbs_find_ports.c @@ -28,7 +28,10 @@ #include #include #include +#include +#include "opal_stdint.h" +#include "opal/types.h" #include "opal/util/output.h" #include "opal/util/argv.h" #include "opal/class/opal_object.h" @@ -157,46 +160,72 @@ static bool want_this_port(char **include_list, char **exclude_list, /* Will never get here */ } +/***********************************************************************/ -/* - * It seems you can't probe a device / port to see if it supports a - * specific type of QP. You just have to try to make it and see if it - * works. This is a short helper function to try to make a QP of a - * specific type and return whether it worked. - */ -static bool make_qp(struct ibv_pd *pd, struct ibv_cq *cq, - enum ibv_qp_type type) +static const char *transport_name_to_str(enum ibv_transport_type transport_type) { - struct ibv_qp_init_attr qpia; - struct ibv_qp *qp; + switch(transport_type) { + case IBV_TRANSPORT_IB: return "IB"; + case IBV_TRANSPORT_IWARP: return "IWARP"; + case IBV_TRANSPORT_UNKNOWN: + default: return "unknown"; + } +} - qpia.qp_context = NULL; - qpia.send_cq = cq; - qpia.recv_cq = cq; - qpia.srq = NULL; - qpia.cap.max_send_wr = 1; - qpia.cap.max_recv_wr = 1; - qpia.cap.max_send_sge = 1; - qpia.cap.max_recv_sge = 1; - qpia.cap.max_inline_data = 0; - qpia.qp_type = type; - qpia.sq_sig_all = 0; - - qp = ibv_create_qp(pd, &qpia); - if (NULL != qp) { - ibv_destroy_qp(qp); - return true; +#if defined(HAVE_IBV_LINK_LAYER_ETHERNET) +static const char *link_layer_to_str(int link_type) +{ + switch(link_type) { + case IBV_LINK_LAYER_INFINIBAND: return "IB"; + case IBV_LINK_LAYER_ETHERNET: return "IWARP"; + case IBV_LINK_LAYER_UNSPECIFIED: + default: return "unspecified"; + } +} +#endif + +/***********************************************************************/ + +static void check_sanity(char ***if_sanity_list, const char *dev_name, int port) +{ + int i; + char tmp[BUFSIZ], **list = *if_sanity_list; + const char *compare; + + if (NULL == if_sanity_list || NULL == *if_sanity_list) { + return; } - return false; + /* A match is found if: + - "dev_name" is in the list and port == -1, or + - "dev_name:port" is in the list + If a match is found, remove that entry from the list. */ + memset(tmp, 0, sizeof(tmp)); + if (port > 0) { + snprintf(tmp, sizeof(tmp) - 1, "%s:%d", dev_name, port); + compare = tmp; + } else { + compare = dev_name; + } + + for (i = 0; NULL != list[i]; ++i) { + if (0 == strcmp(list[i], compare)) { + int count = opal_argv_count(list); + opal_argv_delete(&count, &list, i, 1); + --i; + } + } } +/***********************************************************************/ + /* * Find a list of ibv_ports matching a set of criteria. */ -opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, - const char *if_exclude, - int flags) +opal_list_t *ompi_common_verbs_find_ports(const char *if_include, + const char *if_exclude, + int flags, + int stream) { int32_t num_devs; struct ibv_device **devices; @@ -204,9 +233,7 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, struct ibv_context *device_context; struct ibv_device_attr device_attr; struct ibv_port_attr port_attr; - struct ibv_pd *pd = NULL; - struct ibv_cq *cq = NULL; - char **if_include_list = NULL, **if_exclude_list = NULL; + char **if_include_list = NULL, **if_exclude_list = NULL, **if_sanity_list = NULL; ompi_common_verbs_device_item_t *di; ompi_common_verbs_port_item_t *pi; uint32_t i, j; @@ -224,9 +251,15 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, if (NULL != if_include && NULL != if_exclude) { return port_list; } else if (NULL != if_include) { + opal_output_verbose(5, stream, "finding verbs interfaces, including %s", + if_include); if_include_list = opal_argv_split(if_include, ','); + if_sanity_list = opal_argv_copy(if_include_list); } else if (NULL != if_exclude) { + opal_output_verbose(5, stream, "finding verbs interfaces, excluding %s", + if_exclude); if_exclude_list = opal_argv_split(if_exclude, ','); + if_sanity_list = opal_argv_copy(if_exclude_list); } /* Query all the IBV devices on the machine. Use an ompi @@ -234,15 +267,27 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, over the history of the IBV API. */ devices = ompi_ibv_get_device_list(&num_devs); if (0 == num_devs) { + opal_output_verbose(5, stream, "no verbs interfaces found"); goto err_free_argv; + } else { + opal_output_verbose(5, stream, "found %d verbs interface%s", + num_devs, (num_devs != 1) ? "s" : ""); } /* Now loop through all the devices. Get the attributes for each port on each device to see if they match our selection criteria. */ for (i = 0; (int32_t) i < num_devs; ++i) { + /* See if this device is on the include/exclude sanity check + list. If it is, remove it from the sanity check list + (i.e., we should end up with an empty list at the end if + all entries in the sanity check list exist) */ device = devices[i]; + check_sanity(&if_sanity_list, ibv_get_device_name(device), -1); + device_context = ibv_open_device(device); + opal_output_verbose(5, stream, "examining verbs interface: %s", + ibv_get_device_name(device)); if (NULL == device_context) { orte_show_help("help-ompi-common-verbs.txt", "ibv_open_device fail", true, @@ -261,38 +306,58 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, goto err_free_port_list; } + /* Now that we have the attributes of this device, remove all + ports of this device from the sanity check list. Note that + IBV ports are indexed from 1, not 0. */ + for (j = 1; j <= device_attr.phys_port_cnt; j++) { + check_sanity(&if_sanity_list, ibv_get_device_name(device), j); + } + /* Check the the device-specific flags to see if we want this device */ want = true; if (flags & OMPI_COMMON_VERBS_FLAGS_TRANSPORT_IB && IBV_TRANSPORT_IB != device->transport_type) { + opal_output_verbose(5, stream, "verbs interface %s has wrong type (has %s, want IB)", + ibv_get_device_name(device), + transport_name_to_str(device->transport_type)); want = false; } if (flags & OMPI_COMMON_VERBS_FLAGS_TRANSPORT_IWARP && IBV_TRANSPORT_IWARP != device->transport_type) { + opal_output_verbose(5, stream, "verbs interface %s has wrong type (has %s, want IWARP)", + ibv_get_device_name(device), + transport_name_to_str(device->transport_type)); want = false; } + /* Check for RC or UD QP support */ + if (flags & OMPI_COMMON_VERBS_FLAGS_RC || + flags & OMPI_COMMON_VERBS_FLAGS_UD) { + if (OPAL_SUCCESS != ompi_common_verbs_qp_test(device_context, flags)) { + want = false; + opal_output_verbose(5, stream, + "verbs interface %s:%d: failed to make %s QP", + ibv_get_device_name(device), j, + ((flags & (OMPI_COMMON_VERBS_FLAGS_RC | + OMPI_COMMON_VERBS_FLAGS_UD)) == + (OMPI_COMMON_VERBS_FLAGS_RC | + OMPI_COMMON_VERBS_FLAGS_UD)) ? + "both UD and RC" : + (flags & OMPI_COMMON_VERBS_FLAGS_RC) ? + "RC" : "UD"); + } + } + /* If we didn't want it, go to the next device */ if (!want) { continue; } - /* If we asked for check for RC or UD support, then we'll need - a PD and CQ for checking, below. So alloc one. */ - if (flags & OMPI_COMMON_VERBS_FLAGS_RC || - flags & OMPI_COMMON_VERBS_FLAGS_UD) { - pd = ibv_alloc_pd(device_context); - cq = ibv_create_cq(device_context, 1, NULL, NULL, 0); - if (NULL == cq || NULL == pd) { - goto err_destroy_cq_pd; - } - } - /* Make a device_item_t to hold the device information */ di = OBJ_NEW(ompi_common_verbs_device_item_t); if (NULL == di) { - goto err_destroy_cq_pd; + goto err_free_port_list; } di->device = device; di->context = device_context; @@ -305,6 +370,8 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, /* If we don't want this port (based on if_include / if_exclude lists), skip it */ if (!want_this_port(if_include_list, if_exclude_list, di, j)) { + opal_output_verbose(5, stream, "verbs interface %s:%d: rejected by include/exclude", + ibv_get_device_name(device), j); continue; } @@ -315,11 +382,13 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, orte_process_info.nodename, ibv_get_device_name(device), errno, strerror(errno)); - goto err_destroy_cq_pd; + goto err_free_port_list; } /* We definitely only want ACTIVE ports */ if (IBV_PORT_ACTIVE != port_attr.state) { + opal_output_verbose(5, stream, "verbs interface %s:%d: not ACTIVE", + ibv_get_device_name(device), j); continue; } @@ -329,30 +398,35 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, if (0 == flags) { want = true; } - if (flags & OMPI_COMMON_VERBS_FLAGS_RC) { - /* It doesn't look like you can query whether a - device/port supports RC QP's. You just have to try - to make one. :-( If it succeeds, the device/port - supports it. */ - if (make_qp(pd, cq, IBV_QPT_RC)) { - want = true; - } - } - if (flags & OMPI_COMMON_VERBS_FLAGS_UD) { - /* See above comment about RC QP's -- same rationale holds - true here. */ - if (make_qp(pd, cq, IBV_QPT_UD)) { - want = true; - } - } + + if ((flags & (OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_IB | + OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET)) == + (OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_IB | + OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET)) { + /* If they specified both link layers, then we want this port */ + want = true; + } else if ((flags & (OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_IB | + OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET)) == 0) { + /* If they specified neither link layer, then we want this port */ + want = true; + } #if defined(HAVE_IBV_LINK_LAYER_ETHERNET) - if (flags & OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_IB && - IBV_LINK_LAYER_INFINIBAND == port_attr.link_layer) { - want = true; - } - if (flags & OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET && - IBV_LINK_LAYER_ETHERNET == port_attr.link_layer) { - want = true; + else if (flags & OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_IB) { + if (IBV_LINK_LAYER_INFINIBAND == port_attr.link_layer) { + want = true; + } else { + opal_output_verbose(5, stream, "verbs interface %s:%d has wrong link layer (has %s, want IB)", + ibv_get_device_name(device), j, + link_layer_to_str(port_attr.link_layer)); + } + } else if (flags & OMPI_COMMON_VERBS_FLAGS_LINK_LAYER_ETHERNET) { + if (IBV_LINK_LAYER_ETHERNET == port_attr.link_layer) { + want = true; + } else { + opal_output_verbose(5, stream, "verbs interface %s:%d has wrong link layer (has %s, want Ethernet)", + ibv_get_device_name(device), j, + link_layer_to_str(port_attr.link_layer)); + } } #endif @@ -360,28 +434,20 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, continue; } - /* If we got this far, we want the port. Make an item for - it and add it to the list. */ + /* If we got this far, we want the port. Make an item for it. */ pi = OBJ_NEW(ompi_common_verbs_port_item_t); if (NULL == pi) { - goto err_destroy_cq_pd; + goto err_free_port_list; } pi->device = di; pi->port_num = j; pi->port_attr = port_attr; OBJ_RETAIN(di); + /* Add the port item to the list */ opal_list_append(port_list, &pi->super); - } - - /* If we allocated a pd for testing RC/UD, free it here */ - if (NULL != pd) { - ibv_dealloc_pd(pd); - pd = NULL; - } - if (NULL != cq) { - ibv_destroy_cq(cq); - cq = NULL; + opal_output_verbose(5, stream, "found acceptable verbs interface %s:%d", + ibv_get_device_name(device), j); } /* We're done with the device; if some ports are using it, its @@ -390,19 +456,30 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, OBJ_RELEASE(di); } + /* Sanity check that the devices specified in the if_include / + if_exclude lists actually existed. If this is true, then the + sanity list will now be empty. If there are still items left + on the list, then they didn't exist. Bad. Print a warning (if + the warning is not disabled). */ + if (0 != opal_argv_count(if_sanity_list)) { + if (ompi_common_verbs_warn_nonexistent_if) { + char *str = opal_argv_join(if_sanity_list, ','); + orte_show_help("help-ompi-common-verbs.txt", "nonexistent port", + true, orte_process_info.nodename, + ((NULL != if_include) ? "in" : "ex"), str); + free(str); + + /* Only warn once per process */ + ompi_common_verbs_warn_nonexistent_if = false; + } + } + if (NULL != if_sanity_list) { + opal_argv_free(if_sanity_list); + } + /* All done! */ return port_list; - err_destroy_cq_pd: - if (NULL != pd) { - ibv_dealloc_pd(pd); - pd = NULL; - } - if (NULL != cq) { - ibv_destroy_cq(cq); - cq = NULL; - } - err_free_port_list: for (item = opal_list_remove_first(port_list); item != NULL; @@ -411,6 +488,10 @@ opal_list_t *ompi_common_verbs_find_ibv_ports(const char *if_include, } err_free_argv: + if (NULL != if_sanity_list) { + opal_argv_free(if_sanity_list); + if_sanity_list = NULL; + } opal_argv_free(if_include_list); if_include_list = NULL; opal_argv_free(if_exclude_list); diff --git a/ompi/mca/common/verbs/common_verbs_mca.c b/ompi/mca/common/verbs/common_verbs_mca.c new file mode 100644 index 0000000000..684e69f14a --- /dev/null +++ b/ompi/mca/common/verbs/common_verbs_mca.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include "opal/mca/base/mca_base_param.h" + +#include "common_verbs.h" + +/***********************************************************************/ + +static bool registered = false; +static int warn_nonexistent_if_index = -1; + +bool ompi_common_verbs_warn_nonexistent_if = true; + +static void register_internal(void) +{ + int ival; + + warn_nonexistent_if_index = + mca_base_param_reg_int_name("ompi_common_verbs", + "warn_nonexistent_if", + "Warn if non-existent devices and/or ports are specified in device include/exclude MCA parameters " + "(0 = do not warn; any other value = warn)", + false, false, + (int) ompi_common_verbs_warn_nonexistent_if, + &ival); + ompi_common_verbs_warn_nonexistent_if = (bool) ival; + + registered = true; +} + +void ompi_common_verbs_mca_register(mca_base_component_t *component) +{ + int ival; + + if (!registered) { + register_internal(); + } + + /* Make synonyms for the common_verbs MCA params. Need to look up + the value again, because a new/different value may have been + set by the new synonym name. */ + mca_base_param_reg_syn(warn_nonexistent_if_index, component, + "warn_nonexistent_if", false); + mca_base_param_lookup_int(warn_nonexistent_if_index, &ival); + ompi_common_verbs_warn_nonexistent_if = (bool) ival; +} diff --git a/ompi/mca/common/verbs/common_verbs_qp_type.c b/ompi/mca/common/verbs/common_verbs_qp_type.c new file mode 100644 index 0000000000..c3557546e9 --- /dev/null +++ b/ompi/mca/common/verbs/common_verbs_qp_type.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" + +#include +#include + +#include "ompi/constants.h" + +#include "common_verbs.h" + +/* + * It seems you can't probe a device / port to see if it supports a + * specific type of QP. You just have to try to make it and see if it + * works. This is a short helper function to try to make a QP of a + * specific type and return whether it worked. + */ +static bool make_qp(struct ibv_pd *pd, struct ibv_cq *cq, enum ibv_qp_type type) +{ + struct ibv_qp_init_attr qpia; + struct ibv_qp *qp; + + memset(&qpia, 0, sizeof(qpia)); + qpia.qp_context = NULL; + qpia.send_cq = cq; + qpia.recv_cq = cq; + qpia.srq = NULL; + qpia.cap.max_send_wr = 1; + qpia.cap.max_recv_wr = 1; + qpia.cap.max_send_sge = 1; + qpia.cap.max_recv_sge = 1; + qpia.cap.max_inline_data = 0; + qpia.qp_type = type; + qpia.sq_sig_all = 0; + + qp = ibv_create_qp(pd, &qpia); + if (NULL != qp) { + ibv_destroy_qp(qp); + return true; + } + + return false; +} + +int ompi_common_verbs_qp_test(struct ibv_context *device_context, int flags) +{ + int rc = OMPI_SUCCESS; + struct ibv_pd *pd = NULL; + struct ibv_cq *cq = NULL; + + /* Bozo check */ + if (NULL == device_context || + (0 == (flags & (OMPI_COMMON_VERBS_FLAGS_RC | OMPI_COMMON_VERBS_FLAGS_UD)))) { + return OMPI_ERR_BAD_PARAM; + } + + /* Try to make both the PD and CQ */ + pd = ibv_alloc_pd(device_context); + if (NULL == pd) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + + cq = ibv_create_cq(device_context, 1, NULL, NULL, 0); + if (NULL == cq) { + rc = OMPI_ERR_OUT_OF_RESOURCE; + goto out; + } + + /* Now try to make the QP(s) of the desired type(s) */ + if (flags & OMPI_COMMON_VERBS_FLAGS_RC && + !make_qp(pd, cq, IBV_QPT_RC)) { + rc = OMPI_ERR_NOT_SUPPORTED; + goto out; + } + if (flags & OMPI_COMMON_VERBS_FLAGS_UD && + !make_qp(pd, cq, IBV_QPT_UD)) { + rc = OMPI_ERR_NOT_SUPPORTED; + goto out; + } + + out: + /* Free the PD and/or CQ */ + if (NULL != pd) { + ibv_dealloc_pd(pd); + } + if (NULL != cq) { + ibv_destroy_cq(cq); + } + + return rc; +} diff --git a/ompi/mca/common/verbs/help-ompi-common-verbs.txt b/ompi/mca/common/verbs/help-ompi-common-verbs.txt index da62907ad7..334e976e37 100644 --- a/ompi/mca/common/verbs/help-ompi-common-verbs.txt +++ b/ompi/mca/common/verbs/help-ompi-common-verbs.txt @@ -34,3 +34,15 @@ job may or may not continue. Hostname: %s Device name: %s Errror (%d): %s +# +[nonexistent port] +WARNING: One or more nonexistent OpenFabrics devices/ports were +specified: + + Host: %s + MCA parameter: ompi_common_verbs_%sclude + Nonexistent entities: %s + +These entities will be ignored. You can disable this warning by +setting the ompi_common_verbs_warn_nonexistent_if MCA parameter to 0. +#