btl/usnic: update for mpool/rcache rewrite

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
2015-12-18 09:10:21 -07:00 · 2015-12-18 09:10:21 -07:00 · eac0b110b8
--- a/opal/mca/btl/usnic/btl_usnic.h
+++ b/opal/mca/btl/usnic/btl_usnic.h
@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
 * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
@ -12,6 +13,8 @@
 * Copyright (c) 2006      Sandia National Laboratories. All rights
 *                         reserved.
 * Copyright (c) 2011-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
+ *                         reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
@ -37,7 +40,13 @@
 #include "opal/mca/btl/btl.h"
 #include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/mca/btl/base/base.h"
+#include "opal/mca/rcache/rcache.h"
+
+#include "btl_usnic_compat.h"
+
+#if RCACHE_VERSION < 30
 #include "opal/mca/mpool/grdma/mpool_grdma.h"
+#endif
 #else
 #include "ompi/mca/btl/btl.h"
 #include "ompi/mca/btl/base/btl_base_error.h"
@ -45,8 +54,6 @@
 #include "ompi/mca/mpool/grdma/mpool_grdma.h"
 #endif

-#include "btl_usnic_compat.h"
-
 BEGIN_C_DECLS

 /*
@ -153,8 +160,16 @@ typedef struct opal_btl_usnic_component_t {
    /** list of usnic proc structures */
    opal_list_t usnic_procs;

+#if RCACHE_VERSION == 30
+    /** memory pool hints */
+    char* usnic_mpool_hints;
+
+    /** registration cache name */
+    char *usnic_rcache_name;
+#else
    /** name of memory pool */
    char* usnic_mpool_name;
+#endif

    char *if_include;
    char *if_exclude;
--- a/opal/mca/btl/usnic/btl_usnic_compat.h
+++ b/opal/mca/btl/usnic/btl_usnic_compat.h
@ -1,5 +1,8 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
 * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
@ -8,11 +11,13 @@
 */

 /* This header contains macros to help minimize usnic BTL differences
- * between v1.7/v1.8 and v1.9/v2.0. */
+ * between v1.7/v1.8, v1.9/v2.0, and v2.0/v2.1. */

 #ifndef BTL_USNIC_COMPAT_H
 #define BTL_USNIC_COMPAT_H

+#include "opal/mca/rcache/rcache.h"
+
 /************************************************************************/

 /* v2.0 and beyond */
@ -358,4 +363,18 @@ opal_btl_usnic_put(struct mca_btl_base_module_t *base_module,

 #endif /* BTL_VERSION */

+#if defined(RCACHE_MAJOR_VERSION) && RCACHE_MAJOR_VERSION >= 3
+
+#define RCACHE_VERSION 30
+
+/* these structures got renamed with the mpool/rcache rewrite */
+#define mca_mpool_base_registration_t mca_rcache_base_registration_t
+#define mca_mpool_base_resources_t mca_rcache_base_resources_t
+
+#else
+
+#define RCACHE_VERSION 20
+
+#endif
+
 #endif /* BTL_USNIC_COMPAT_H */
--- a/opal/mca/btl/usnic/btl_usnic_mca.c
+++ b/opal/mca/btl/usnic/btl_usnic_mca.c
@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
 * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
@ -12,7 +13,7 @@
 * Copyright (c) 2006      Sandia National Laboratories. All rights
 *                         reserved.
 * Copyright (c) 2008-2016 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2012-2016 Los Alamos National Security, LLC.  All rights
 *                         reserved.
 * Copyright (c) 2015      Intel, Inc. All rights reserved.
 * $COPYRIGHT$
@ -203,9 +204,19 @@ int opal_btl_usnic_component_register(void)
                  0, &stats_relative, 0, OPAL_INFO_LVL_4));
    mca_btl_usnic_component.stats_relative = (bool) stats_relative;

+#if RCACHE_VERSION == 30
+    CHECK(reg_string("mpool_hints", "Hints to use when selecting mpool",
+                     NULL, &mca_btl_usnic_component.usnic_mpool_hints, 0,
+                     OPAL_INFO_LVL_5));
+
+    CHECK(reg_string("rcache", "Name of the registration cache to be used",
+                     "grdma", &mca_btl_usnic_component.usnic_rcache_name, 0,
+                     OPAL_INFO_LVL_5));
+#else
    CHECK(reg_string("mpool", "Name of the memory pool to be used",
                     "grdma", &mca_btl_usnic_component.usnic_mpool_name, 0,
                     OPAL_INFO_LVL_5));
+#endif

    want_numa_device_assignment = OPAL_HAVE_HWLOC ? 1 : -1;
    CHECK(reg_int("want_numa_device_assignment",
--- a/opal/mca/btl/usnic/btl_usnic_module.c
+++ b/opal/mca/btl/usnic/btl_usnic_module.c
@ -13,7 +13,7 @@
 * Copyright (c) 2006      Sandia National Laboratories. All rights
 *                         reserved.
 * Copyright (c) 2009-2016 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
 *                         reserved.
 * Copyright (c) 2014      Intel, Inc. All rights reserved
 * $COPYRIGHT$
@ -44,6 +44,8 @@
 #include "opal/mca/btl/base/btl_base_error.h"
 #include "opal/mca/mpool/base/base.h"
 #include "opal/mca/mpool/mpool.h"
+#include "opal/mca/rcache/base/base.h"
+#include "opal/mca/rcache/rcache.h"
 #else
 #include "ompi/mca/btl/btl.h"
 #include "ompi/mca/btl/base/btl_base_error.h"
@ -884,7 +886,11 @@ static int usnic_finalize(struct mca_btl_base_module_t* btl)
    OBJ_DESTRUCT(&module->chunk_segs);
    OBJ_DESTRUCT(&module->senders);

+#if RCACHE_VERSION == 30
+    mca_rcache_base_module_destroy(module->rcache);
+#else
    mca_mpool_base_module_destroy(module->super.btl_mpool);
+#endif

    if (NULL != module->av) {
        fi_close(&module->av->fid);
@ -1736,9 +1742,9 @@ static int init_one_channel(opal_btl_usnic_module_t *module,
                                    rd_num /* num erorments to alloc */,
                                    rd_num /* max elements to alloc */,
                                    rd_num /* num elements per alloc */,
-                                    module->super.btl_mpool /* mpool for reg */,
+                                    module->super.btl_mpool /* mpool for (1.x, 2.0: reg, 2.1+: allocation) */,
                                    0 /* mpool reg flags */,
-                                    NULL /* unused0 */,
+                                    module->rcache /* registration cache for 2.1+ */,
                                    NULL /* item_init */,
                                    NULL /* item_init_context */);
    channel->recv_segs.ctx = module; /* must come after
@ -2054,11 +2060,28 @@ static int init_mpool(opal_btl_usnic_module_t *module)
    mpool_resources.sizeof_reg = sizeof(opal_btl_usnic_reg_t);
    mpool_resources.register_mem = usnic_reg_mr;
    mpool_resources.deregister_mem = usnic_dereg_mr;
+#if RCACHE_VERSION == 30
+    module->rcache =
+        mca_rcache_base_module_create (mca_btl_usnic_component.usnic_rcache_name,
+                                       &module->super, &mpool_resources);
+    if (NULL == module->rcache) {
+        opal_show_help("help-mpi-btl-usnic.txt",
+                       "internal error during init",
+                       true,
+                       opal_process_info.nodename,
+                       module->fabric_info->fabric_attr->name,
+                       "create rcache", __FILE__, __LINE__);
+        return OPAL_ERROR;
+    }
+    module->super.btl_mpool =
+        mca_mpool_base_module_lookup (mca_btl_usnic_component.usnic_mpool_hints);
+#else
    asprintf(&mpool_resources.pool_name, "%s",
             module->fabric_info->fabric_attr->name);
    module->super.btl_mpool =
        mca_mpool_base_module_create(mca_btl_usnic_component.usnic_mpool_name,
                                     &module->super, &mpool_resources);
+#endif
    if (NULL == module->super.btl_mpool) {
        opal_show_help("help-mpi-btl-usnic.txt",
                       "internal error during init",
@ -2232,7 +2255,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
                             module->sd_num / 2,
                             module->super.btl_mpool,
                             0 /* mpool reg flags */,
-                             NULL /* unused0 */,
+                             module->rcache,
                             NULL /* item_init */,
                             NULL /* item_init_context */);
    assert(OPAL_SUCCESS == rc);
@ -2287,7 +2310,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
                             module->sd_num / 2,
                             module->super.btl_mpool,
                             0 /* mpool reg flags */,
-                             NULL /* unused0 */,
+                             module->rcache,
                             NULL /* item_init */,
                             NULL /* item_init_context */);
    assert(OPAL_SUCCESS == rc);
@ -2309,7 +2332,7 @@ static void init_freelists(opal_btl_usnic_module_t *module)
                             module->sd_num / 2,
                             module->super.btl_mpool,
                             0 /* mpool reg flags */,
-                             NULL /* unused0 */,
+                             module->rcache,
                             NULL /* item_init */,
                             NULL /* item_init_context */);
    assert(OPAL_SUCCESS == rc);
@ -2366,7 +2389,11 @@ int opal_btl_usnic_module_init(opal_btl_usnic_module_t *module)
    int ret;
    if (OPAL_SUCCESS != (ret = init_mpool(module)) ||
        OPAL_SUCCESS != (ret = init_channels(module))) {
+#if RCACHE_VERSION == 30
+        mca_rcache_base_module_destroy (module->rcache);
+#else
        mca_mpool_base_module_destroy(module->super.btl_mpool);
+#endif
        return ret;
    }

--- a/opal/mca/btl/usnic/btl_usnic_module.h
+++ b/opal/mca/btl/usnic/btl_usnic_module.h
@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
 * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
@ -12,6 +13,8 @@
 * Copyright (c) 2006      Sandia National Laboratories. All rights
 *                         reserved.
 * Copyright (c) 2011-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
+ *                         reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
@ -197,6 +200,9 @@ typedef struct opal_btl_usnic_module_t {

    /* Performance / debugging statistics */
    opal_btl_usnic_module_stats_t stats;
+
+    /** registration cache module (v2.1+) */
+    mca_rcache_base_module_t *rcache;
 } opal_btl_usnic_module_t;

 struct opal_btl_usnic_frag_t;