125d236173
We've been fighting the battle of trying to create a regex generator and parser that can handle arbitrary hostname schemes - without long-term success. The worst of it is that there is no way of checking to see if the computed regex is correct short of parsing it and doing a character-by-character comparison with the original string. Ugh...there has to be a better solution. One option is to investigate using 3rd-party regex libraries as those are coming from communities whose sole focus is resolving that problem. However, someone would need to spend the time to investigate it, and we'd have to find a license-friendly implementation. Another option is to quit beating our heads against the wall and just compress the information. It won't be as much of a reduction, but we also won't keep hitting scenarios where things break. In this case, it seems that "perfection" is definitely the enemy of "good enough". This PR implements the compression option while retaining the possibility of people adding regex-generating components. The compression code used in ORTE is consolidated into the opal/compress framework. That framework currently held bzip and gzip components for use in compressing checkpoint files - since we no longer support C/R, I have .opal_ignore'd those components. However, I have left the original framework APIs alone in case someone ever decides to redo C/R. The APIs of interest here are added to the framework - specifically, the "compress_block" and "decompress_block" functions. I then moved the ORTE zlib compression code into a new component in this framework. Unfortunately, the framework currently is a single-select one - i.e., only one active component at a time. Since I .opal_ignore'd the other two and made the priority of zlib high, this isn't a problem. However, if someone wants to re-enable bzip/gzip or add another component, they might need to transition opal/compress to a multi-select framework. Included changes: * Consolidate the compression code into the opal/compress framework * Move the ORTE zlib compression code into a new opal/compress/zlib component * Ignore the bzip and gzip components in opal/compress framework * Add a "compress_base_limit" MCA param to set the threshold above which we compress data - defaults to 4096 bytes * Delete stale brucks and rcd components from orte/grpcomm framework * Delete the orte/regx framework * Update the launch system to use opal/compress instead of string regex * Provide a default module if no zlib is available * Fix some misc multi-node issues * Properly generate the nidmap in response to a "connection warmup" message so the remote daemon knows the children it needs to launch. * Remove stale references to orte_node_regex * opal_byte_object_t's are not OPAL objects - properly release allocated memory. * Set the topology * Currently only handling homogeneous case * Update the compress framework files to conform * Consolidate open/close into one "frame" file. Ensure we open/close the framework Signed-off-by: Ralph Castain <rhc@pmix.org>
157 строки
4.5 KiB
C
157 строки
4.5 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
*
|
|
* Copyright (c) 2019 Intel, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
/**
|
|
* @file
|
|
*
|
|
* Compression Framework
|
|
*
|
|
* General Description:
|
|
*
|
|
* The OPAL Compress framework has been created to provide an abstract interface
|
|
* to the compression agent library on the host machine. This fromework is useful
|
|
* when distributing files that can be compressed before sending to dimish the
|
|
* load on the network.
|
|
*
|
|
*/
|
|
|
|
#ifndef MCA_COMPRESS_H
|
|
#define MCA_COMPRESS_H
|
|
|
|
#include "opal_config.h"
|
|
#include "opal/mca/mca.h"
|
|
#include "opal/mca/base/base.h"
|
|
#include "opal/class/opal_object.h"
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
/**
|
|
* Module initialization function.
|
|
* Returns OPAL_SUCCESS
|
|
*/
|
|
typedef int (*opal_compress_base_module_init_fn_t)
|
|
(void);
|
|
|
|
/**
|
|
* Module finalization function.
|
|
* Returns OPAL_SUCCESS
|
|
*/
|
|
typedef int (*opal_compress_base_module_finalize_fn_t)
|
|
(void);
|
|
|
|
/**
|
|
* Compress the file provided
|
|
*
|
|
* Arguments:
|
|
* fname = Filename to compress
|
|
* cname = Compressed filename
|
|
* postfix = postfix added to filename to create compressed filename
|
|
* Returns:
|
|
* OPAL_SUCCESS on success, ow OPAL_ERROR
|
|
*/
|
|
typedef int (*opal_compress_base_module_compress_fn_t)
|
|
(char * fname, char **cname, char **postfix);
|
|
|
|
typedef int (*opal_compress_base_module_compress_nb_fn_t)
|
|
(char * fname, char **cname, char **postfix, pid_t *child_pid);
|
|
|
|
/**
|
|
* Decompress the file provided
|
|
*
|
|
* Arguments:
|
|
* fname = Filename to compress
|
|
* cname = Compressed filename
|
|
* Returns:
|
|
* OPAL_SUCCESS on success, ow OPAL_ERROR
|
|
*/
|
|
typedef int (*opal_compress_base_module_decompress_fn_t)
|
|
(char * cname, char **fname);
|
|
typedef int (*opal_compress_base_module_decompress_nb_fn_t)
|
|
(char * cname, char **fname, pid_t *child_pid);
|
|
|
|
/**
|
|
* Compress a string
|
|
*
|
|
* Arguments:
|
|
*
|
|
*/
|
|
typedef bool (*opal_compress_base_module_compress_string_fn_t)(uint8_t *inbytes,
|
|
size_t inlen,
|
|
uint8_t **outbytes,
|
|
size_t *olen);
|
|
typedef bool (*opal_compress_base_module_decompress_string_fn_t)(uint8_t **outbytes, size_t olen,
|
|
uint8_t *inbytes, size_t len);
|
|
|
|
|
|
/**
|
|
* Structure for COMPRESS components.
|
|
*/
|
|
struct opal_compress_base_component_2_0_0_t {
|
|
/** MCA base component */
|
|
mca_base_component_t base_version;
|
|
/** MCA base data */
|
|
mca_base_component_data_t base_data;
|
|
|
|
/** Verbosity Level */
|
|
int verbose;
|
|
/** Output Handle for opal_output */
|
|
int output_handle;
|
|
/** Default Priority */
|
|
int priority;
|
|
};
|
|
typedef struct opal_compress_base_component_2_0_0_t opal_compress_base_component_2_0_0_t;
|
|
typedef struct opal_compress_base_component_2_0_0_t opal_compress_base_component_t;
|
|
|
|
/**
|
|
* Structure for COMPRESS modules
|
|
*/
|
|
struct opal_compress_base_module_1_0_0_t {
|
|
/** Initialization Function */
|
|
opal_compress_base_module_init_fn_t init;
|
|
/** Finalization Function */
|
|
opal_compress_base_module_finalize_fn_t finalize;
|
|
|
|
/** Compress interface */
|
|
opal_compress_base_module_compress_fn_t compress;
|
|
opal_compress_base_module_compress_nb_fn_t compress_nb;
|
|
|
|
/** Decompress Interface */
|
|
opal_compress_base_module_decompress_fn_t decompress;
|
|
opal_compress_base_module_decompress_nb_fn_t decompress_nb;
|
|
|
|
/* COMPRESS STRING */
|
|
opal_compress_base_module_compress_string_fn_t compress_block;
|
|
opal_compress_base_module_decompress_string_fn_t decompress_block;
|
|
};
|
|
typedef struct opal_compress_base_module_1_0_0_t opal_compress_base_module_1_0_0_t;
|
|
typedef struct opal_compress_base_module_1_0_0_t opal_compress_base_module_t;
|
|
|
|
OPAL_DECLSPEC extern opal_compress_base_module_t opal_compress;
|
|
|
|
/**
|
|
* Macro for use in components that are of type COMPRESS
|
|
*/
|
|
#define OPAL_COMPRESS_BASE_VERSION_2_0_0 \
|
|
OPAL_MCA_BASE_VERSION_2_1_0("compress", 2, 0, 0)
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
}
|
|
#endif
|
|
|
|
#endif /* OPAL_COMPRESS_H */
|
|
|