openmpi/opal/mca/btl/self/btl_self_component.c

/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
 * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
 *                         Corporation.  All rights reserved.
 * Copyright (c) 2004-2006 The University of Tennessee and The University
 *                         of Tennessee Research Foundation.  All rights
 *                         reserved.
 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 *                         University of Stuttgart.  All rights reserved.
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
 * Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
 *                         reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */
#include "opal_config.h"

#include "btl_self.h"
#include "btl_self_frag.h"

static int mca_btl_self_component_register(void);
static int mca_btl_self_component_open(void);
static int mca_btl_self_component_close(void);

/**
 * SELF module initialization.
 *
 * @param num_btls (OUT)                  Number of BTLs returned in BTL array.
 * @param enable_progress_threads (IN)    Flag indicating whether BTL is allowed to have progress threads
 * @param enable_mpi_threads (IN)         Flag indicating whether BTL must support multilple simultaneous invocations from different threads
 *
 */
static mca_btl_base_module_t **mca_btl_self_component_init (int *num_btls,
                                                            bool enable_progress_threads,
                                                            bool enable_mpi_threads);

/*
 * Shared Memory (SELF) component instance.
 */

mca_btl_self_component_t mca_btl_self_component = {
    .super = {
        /* First, the mca_base_component_t struct containing meta information
          about the component itself */
        .btl_version = {
            MCA_BTL_DEFAULT_VERSION("self"),
            .mca_open_component = mca_btl_self_component_open,
            .mca_close_component = mca_btl_self_component_close,
            .mca_register_component_params = mca_btl_self_component_register,
        },
        .btl_data = {
            /* The component is checkpoint ready */
            .param_field = MCA_BASE_METADATA_PARAM_CHECKPOINT,
        },

        .btl_init = mca_btl_self_component_init,
    }  /* end super */
};

/*
 *  Called by MCA framework to open the component, registers
 *  component parameters.
 */

static int mca_btl_self_component_register(void)
{
    mca_base_var_group_component_register(&mca_btl_self_component.super.btl_version,
                                          "BTL for self communication");

    /* register SELF component parameters */
    mca_btl_self_component.free_list_num = 0;
    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_num",
                                           "Number of fragments by default",
                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
                                           OPAL_INFO_LVL_9,
                                           MCA_BASE_VAR_SCOPE_READONLY,
                                           &mca_btl_self_component.free_list_num);
    /* NTH: free list buffers are not released until we tear down so DO NOT make them unlimited here */
    mca_btl_self_component.free_list_max = 64;
    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_max",
                                           "Maximum number of fragments",
                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
                                           OPAL_INFO_LVL_9,
                                           MCA_BASE_VAR_SCOPE_READONLY,
                                           &mca_btl_self_component.free_list_max);
    mca_btl_self_component.free_list_inc = 8;
    (void) mca_base_component_var_register(&mca_btl_self_component.super.btl_version, "free_list_inc",
                                           "Increment by this number of fragments",
                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
                                           OPAL_INFO_LVL_9,
                                           MCA_BASE_VAR_SCOPE_READONLY,
                                           &mca_btl_self_component.free_list_inc);

    mca_btl_self.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH;
    mca_btl_self.btl_eager_limit = 1024;
    mca_btl_self.btl_rndv_eager_limit = 128 * 1024;
    mca_btl_self.btl_max_send_size = 16 * 1024;
    mca_btl_self.btl_rdma_pipeline_send_length = INT_MAX;
    mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX;
    mca_btl_self.btl_min_rdma_pipeline_size = 0;
    mca_btl_self.btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND;
    mca_btl_self.btl_bandwidth = 100;
    mca_btl_self.btl_latency = 0;
    mca_btl_base_param_register (&mca_btl_self_component.super.btl_version, &mca_btl_self);

    return OPAL_SUCCESS;
}

static int mca_btl_self_component_open(void)
{
    /* initialize objects */
    OBJ_CONSTRUCT(&mca_btl_self_component.self_frags_eager, opal_free_list_t);
    OBJ_CONSTRUCT(&mca_btl_self_component.self_frags_send, opal_free_list_t);
    OBJ_CONSTRUCT(&mca_btl_self_component.self_frags_rdma, opal_free_list_t);

    return OPAL_SUCCESS;
}


/*
 * component cleanup - sanity checking of queue lengths
 */

static int mca_btl_self_component_close(void)
{
    OBJ_DESTRUCT(&mca_btl_self_component.self_frags_eager);
    OBJ_DESTRUCT(&mca_btl_self_component.self_frags_send);
    OBJ_DESTRUCT(&mca_btl_self_component.self_frags_rdma);
    return OPAL_SUCCESS;
}

/*
 *  SELF component initialization
 */
static mca_btl_base_module_t **mca_btl_self_component_init (int *num_btls,
                                                            bool enable_progress_threads,
                                                            bool enable_mpi_threads)
{
    mca_btl_base_module_t **btls = NULL;
    int ret;

    /* initialize free lists */
    ret = opal_free_list_init (&mca_btl_self_component.self_frags_eager,
                               sizeof (mca_btl_self_frag_eager_t) + mca_btl_self.btl_eager_limit,
                               opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_eager_t), 0,
                               opal_cache_line_size, mca_btl_self_component.free_list_num,
                               mca_btl_self_component.free_list_max,
                               mca_btl_self_component.free_list_inc,
                               NULL, 0, NULL, NULL, NULL);
    if (OPAL_SUCCESS != ret) {
        return NULL;
    }

    ret = opal_free_list_init (&mca_btl_self_component.self_frags_send,
                               sizeof (mca_btl_self_frag_send_t) + mca_btl_self.btl_max_send_size,
                               opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_send_t), 0,
                               opal_cache_line_size, mca_btl_self_component.free_list_num,
                               mca_btl_self_component.free_list_max,
                               mca_btl_self_component.free_list_inc,
                               NULL, 0, NULL, NULL, NULL);
    if (OPAL_SUCCESS != ret) {
        return NULL;
    }

    ret = opal_free_list_init (&mca_btl_self_component.self_frags_rdma,
                               sizeof (mca_btl_self_frag_rdma_t) + MCA_BTL_SELF_MAX_INLINE_SIZE,
                               opal_cache_line_size, OBJ_CLASS(mca_btl_self_frag_rdma_t), 0,
                               opal_cache_line_size, mca_btl_self_component.free_list_num,
                               mca_btl_self_component.free_list_max,
                               mca_btl_self_component.free_list_inc,
                               NULL, 0, NULL, NULL, NULL);
    if (OPAL_SUCCESS != ret) {
        return NULL;
    }

    /* get pointer to the btls */
    btls = (mca_btl_base_module_t **) malloc (sizeof (mca_btl_base_module_t *));
    if (NULL == btls) {
        return NULL;
    }

    btls[0] = &mca_btl_self;
    *num_btls = 1;

    return btls;
}