Bring tree back to ability to cleanly compile. These changes should be transparent to virtually everyone, but the silly tree insists on compiling everything. Mostly modified things in preparation for releasing the openmpi program, and to begin knitting the next (hopefully final) version of mpirun.
Let me know if it has any impact on you - it shouldn't. This commit was SVN r2487.
Этот коммит содержится в:
родитель
db2e993a3d
Коммит
3c94af6021
@ -26,7 +26,10 @@ enum {
|
||||
OMPI_ERR_NOT_FOUND = -16,
|
||||
OMPI_ERR_BUFFER = -17, /* equivalent to MPI_ERR_BUFFER */
|
||||
OMPI_ERR_REQUEST = -18, /* equivalent to MPI_ERR_REQUEST */
|
||||
OMPI_EXISTS = -20 /* indicates that the specified object already exists */
|
||||
OMPI_EXISTS = -19, /* indicates that the specified object already exists */
|
||||
OMPI_ERR_NO_CONNECTION_ALLOWED = -20, /* indicates that the receiving process does not allow connections */
|
||||
OMPI_ERR_CONNECTION_REFUSED = -21, /* contact made with process, but it refuses any further communication */
|
||||
OMPI_ERR_CONNECTION_FAILED = -22 /* message sent, but delivery failed */
|
||||
};
|
||||
|
||||
#endif /* OMPI_CONSTANTS_H */
|
||||
|
@ -28,6 +28,9 @@ libruntime_la_SOURCES = \
|
||||
ompi_rte_init.c \
|
||||
ompi_rte_llm.c \
|
||||
ompi_rte_monitor.c \
|
||||
ompi_rte_cmd_line_setup.c \
|
||||
ompi_rte_universe_exists.c \
|
||||
ompi_rte_parse_seed_cmd_line.c \
|
||||
ompi_rte_pcm.c
|
||||
|
||||
# Conditionally install the header files
|
||||
|
@ -17,6 +17,7 @@
|
||||
#include "mca/pcmclient/base/base.h"
|
||||
#include "mca/oob/oob.h"
|
||||
#include "mca/ns/base/base.h"
|
||||
#include "mca/gpr/base/base.h"
|
||||
#include "util/session_dir.h"
|
||||
|
||||
/**
|
||||
@ -37,6 +38,7 @@ int ompi_rte_finalize(void)
|
||||
mca_llm_base_close();
|
||||
mca_pcmclient_base_close();
|
||||
mca_ns_base_close();
|
||||
mca_gpr_base_close();
|
||||
mca_oob_base_close();
|
||||
|
||||
ompi_session_dir_finalize();
|
||||
|
@ -89,7 +89,22 @@
|
||||
*/
|
||||
|
||||
/* globals used by RTE */
|
||||
int ompi_rte_debug_flag;
|
||||
int ompi_rte_debug_flag=0;
|
||||
ompi_universe_t ompi_universe_info = {
|
||||
/* .name = */ "default-universe",
|
||||
/* .host = */ "localhost",
|
||||
/* .uid = */ NULL,
|
||||
/* .persistence = */ false,
|
||||
/* .scope = */ "local",
|
||||
/* .silent_mode = */ true,
|
||||
/* .script_mode = */ false,
|
||||
/* .web_server = */ false,
|
||||
/* .socket_contact_info = */ NULL,
|
||||
/* .oob_contact_info = */ NULL,
|
||||
/* .console_connected = */ false,
|
||||
/* .scriptfile = */ NULL,
|
||||
/* .hostfile = */ NULL
|
||||
};
|
||||
|
||||
|
||||
int ompi_rte_init(bool *allow_multi_user_threads, bool *have_hidden_threads)
|
||||
|
@ -10,11 +10,12 @@
|
||||
#include "mca/pcm/pcm.h"
|
||||
|
||||
extern mca_pcm_base_module_t mca_pcm;
|
||||
|
||||
ompi_list_t*
|
||||
ompi_rte_allocate_resources(int jobid, int nodes, int procs)
|
||||
{
|
||||
if (NULL == mca_pcm.pcm_allocate_resources) {
|
||||
return OMPI_ERROR;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return mca_pcm.pcm_allocate_resources(jobid, nodes, procs);
|
||||
|
@ -71,6 +71,8 @@ int ompi_rte_register(void)
|
||||
|
||||
ompi_buffer_free(buffer);
|
||||
return rc;
|
||||
} else if (ompi_rte_debug_flag) {
|
||||
ompi_output(0, "rte_register: oob does NOT have seed");
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
@ -27,6 +27,28 @@ extern "C" {
|
||||
|
||||
extern int ompi_rte_debug_flag;
|
||||
|
||||
/* Define the info structure underlying the Open MPI universe system
|
||||
* instanced in ompi_rte_init.c */
|
||||
|
||||
struct ompi_universe_t {
|
||||
char *name;
|
||||
char *host;
|
||||
char *uid;
|
||||
bool persistence;
|
||||
char *scope;
|
||||
bool silent_mode;
|
||||
bool script_mode;
|
||||
bool web_server;
|
||||
char *socket_contact_info;
|
||||
char *oob_contact_info;
|
||||
bool console_connected;
|
||||
char *scriptfile;
|
||||
char *hostfile;
|
||||
};
|
||||
typedef struct ompi_universe_t ompi_universe_t;
|
||||
|
||||
extern ompi_universe_t ompi_universe_info;
|
||||
|
||||
/**
|
||||
* Initialize the Open MPI support code
|
||||
*
|
||||
@ -215,6 +237,39 @@ extern "C" {
|
||||
*/
|
||||
void ompi_rte_cmd_line_setup(ompi_cmd_line_t *cmd_line);
|
||||
|
||||
|
||||
/**
|
||||
* Parse the rte command line for options
|
||||
*
|
||||
* Parses the specified command line for rte/seed daemon specific options.
|
||||
* Fills the relevant global structures with the information obtained.
|
||||
*
|
||||
* @param cmd_line Command line to be parsed.
|
||||
* @retval None
|
||||
*/
|
||||
void ompi_rte_parse_seed_cmd_line(ompi_cmd_line_t *cmd_line);
|
||||
|
||||
/**
|
||||
* Check for universe existence
|
||||
*
|
||||
* Checks to see if a specified universe exists. If so, attempts
|
||||
* to connect to verify that the universe is accepting connections.
|
||||
*
|
||||
* @param None Reads everything from the process_info and system_info
|
||||
* structures
|
||||
*
|
||||
* @retval OMPI_SUCCESS Universe found and connection accepted
|
||||
* @retval OMPI_NO_CONNECTION_ALLOWED Universe found, but not persistent or
|
||||
* restricted to local scope
|
||||
* @retval OMPI_CONNECTION_FAILED Universe found, but connection attempt
|
||||
* failed. Probably caused by unclean termination of the universe seed
|
||||
* daemon.
|
||||
* @retval OMPI_CONNECTION_REFUSED Universe found and contact made, but
|
||||
* universe refused to allow connection.
|
||||
*/
|
||||
int ompi_rte_universe_exists(char *host, char *name, char *tmpdir,
|
||||
char *oob_contact_inf);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
@ -44,142 +44,156 @@ int main(int argc, char *argv[])
|
||||
|
||||
ompi_cmd_line_t *mca_cmd_line=NULL;
|
||||
|
||||
/*
|
||||
* Intialize the Open MPI environment
|
||||
*/
|
||||
if (OMPI_SUCCESS != ompi_init(argc, argv)) {
|
||||
/* BWB show_help */
|
||||
printf("show_help: ompi_init failed\n");
|
||||
return ret;
|
||||
}
|
||||
/* /\* */
|
||||
/* * Intialize the Open MPI environment */
|
||||
/* *\/ */
|
||||
/* if (OMPI_SUCCESS != ompi_init(argc, argv)) { */
|
||||
/* /\* BWB show_help *\/ */
|
||||
/* printf("show_help: ompi_init failed\n"); */
|
||||
/* return ret; */
|
||||
/* } */
|
||||
|
||||
/* get the system info */
|
||||
ompi_sys_info();
|
||||
/* /\* get the system info *\/ */
|
||||
/* ompi_sys_info(); */
|
||||
|
||||
/* setup to read common command line options that span all Open MPI programs */
|
||||
if (OMPI_SUCCESS != (ret = ompi_common_cmd_line_init(argc, argv))) {
|
||||
exit(ret);
|
||||
}
|
||||
/* /\* setup to read common command line options that span all Open MPI programs *\/ */
|
||||
/* if (OMPI_SUCCESS != (ret = ompi_common_cmd_line_init(argc, argv))) { */
|
||||
/* exit(ret); */
|
||||
/* } */
|
||||
|
||||
if (ompi_cmd_line_is_taken(ompi_common_cmd_line, "help") ||
|
||||
ompi_cmd_line_is_taken(ompi_common_cmd_line, "h")) {
|
||||
printf("...showing ompi_info help message...\n");
|
||||
exit(1);
|
||||
}
|
||||
/* if (ompi_cmd_line_is_taken(ompi_common_cmd_line, "help") || */
|
||||
/* ompi_cmd_line_is_taken(ompi_common_cmd_line, "h")) { */
|
||||
/* printf("...showing ompi_info help message...\n"); */
|
||||
/* exit(1); */
|
||||
/* } */
|
||||
|
||||
/* setup the rte command line arguments */
|
||||
cmd_line = OBJ_NEW(ompi_cmd_line_t);
|
||||
ompi_cmd_line_make_opt(cmd_line, 's', "seed", 0,
|
||||
"Set the daemon seed to true.");
|
||||
/* /\* setup the rte command line arguments *\/ */
|
||||
/* cmd_line = OBJ_NEW(ompi_cmd_line_t); */
|
||||
/* ompi_cmd_line_make_opt(cmd_line, 's', "seed", 0, */
|
||||
/* "Set the daemon seed to true."); */
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line,
|
||||
'u', "universe", 1,
|
||||
"Specify the Open MPI universe");
|
||||
/* ompi_cmd_line_make_opt(cmd_line, */
|
||||
/* 'u', "universe", 1, */
|
||||
/* "Specify the Open MPI universe"); */
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line,
|
||||
't', "tmpdir", 1,
|
||||
"Specify the Open MPI prefix for the session directory");
|
||||
/* ompi_cmd_line_make_opt(cmd_line, */
|
||||
/* 't', "tmpdir", 1, */
|
||||
/* "Specify the Open MPI prefix for the session directory"); */
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line, 'w', "webserver", 0,
|
||||
"Web server available");
|
||||
/* ompi_cmd_line_make_opt(cmd_line, 'w', "webserver", 0, */
|
||||
/* "Web server available"); */
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line, 's', "silent", 0,
|
||||
"No console prompt - operate silently");
|
||||
/* ompi_cmd_line_make_opt(cmd_line, 's', "silent", 0, */
|
||||
/* "No console prompt - operate silently"); */
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line, 'f', "script", 1,
|
||||
"Read commands from script file");
|
||||
/* ompi_cmd_line_make_opt(cmd_line, 'f', "script", 1, */
|
||||
/* "Read commands from script file"); */
|
||||
|
||||
/*
|
||||
* setup mca command line arguments
|
||||
*/
|
||||
mca_cmd_line = OBJ_NEW(ompi_cmd_line_t);
|
||||
if (OMPI_SUCCESS != (ret = mca_base_cmd_line_setup(mca_cmd_line))) {
|
||||
/* BWB show_help */
|
||||
printf("show_help: mca_base_cmd_line_setup failed\n");
|
||||
return ret;
|
||||
}
|
||||
/* /\* */
|
||||
/* * setup mca command line arguments */
|
||||
/* *\/ */
|
||||
/* mca_cmd_line = OBJ_NEW(ompi_cmd_line_t); */
|
||||
/* if (OMPI_SUCCESS != (ret = mca_base_cmd_line_setup(mca_cmd_line))) { */
|
||||
/* /\* BWB show_help *\/ */
|
||||
/* printf("show_help: mca_base_cmd_line_setup failed\n"); */
|
||||
/* return ret; */
|
||||
/* } */
|
||||
|
||||
if (OMPI_SUCCESS != mca_base_cmd_line_process_args(mca_cmd_line)) {
|
||||
/* BWB show_help */
|
||||
printf("show_help: mca_base_cmd_line_process_args\n");
|
||||
return ret;
|
||||
}
|
||||
/* if (OMPI_SUCCESS != mca_base_cmd_line_process_args(mca_cmd_line)) { */
|
||||
/* /\* BWB show_help *\/ */
|
||||
/* printf("show_help: mca_base_cmd_line_process_args\n"); */
|
||||
/* return ret; */
|
||||
/* } */
|
||||
|
||||
if (OMPI_SUCCESS != ompi_cmd_line_parse(cmd_line, true, argc, argv)) {
|
||||
exit(ret);
|
||||
}
|
||||
/* if (OMPI_SUCCESS != ompi_cmd_line_parse(cmd_line, true, argc, argv)) { */
|
||||
/* exit(ret); */
|
||||
/* } */
|
||||
|
||||
/* Do the parsing */
|
||||
/* /\* Do the parsing *\/ */
|
||||
|
||||
if (ompi_cmd_line_is_taken(ompi_common_cmd_line, "help")) {
|
||||
#if 1
|
||||
printf("...showing ompid help message...\n");
|
||||
printf("\nThe following optional arguments are available: --v --h --seeded\n");
|
||||
printf("\t ompid -h is used to display this information\n");
|
||||
printf("\t ompid -v is used to display the version of this application\n");
|
||||
printf("\t ompid -seed is used to start the universe (seed) daemon.\n");
|
||||
printf("\n");
|
||||
#else
|
||||
show_help("ompid", "usage", NULL);
|
||||
#endif
|
||||
exit(1);
|
||||
}
|
||||
/* if (ompi_cmd_line_is_taken(ompi_common_cmd_line, "help")) { */
|
||||
/* #if 1 */
|
||||
/* printf("...showing ompid help message...\n"); */
|
||||
/* printf("\nThe following optional arguments are available: --v --h --seeded\n"); */
|
||||
/* printf("\t ompid -h is used to display this information\n"); */
|
||||
/* printf("\t ompid -v is used to display the version of this application\n"); */
|
||||
/* printf("\t ompid -seed is used to start the universe (seed) daemon.\n"); */
|
||||
/* printf("\n"); */
|
||||
/* #else */
|
||||
/* show_help("ompid", "usage", NULL); */
|
||||
/* #endif */
|
||||
/* exit(1); */
|
||||
/* } */
|
||||
|
||||
|
||||
if (OMPI_SUCCESS != (ret = mca_base_open())) {
|
||||
/* JMS show_help */
|
||||
printf("show_help: mca_base_open failed\n");
|
||||
return ret;
|
||||
}
|
||||
/* if (OMPI_SUCCESS != (ret = mca_base_open())) { */
|
||||
/* /\* JMS show_help *\/ */
|
||||
/* printf("show_help: mca_base_open failed\n"); */
|
||||
/* return ret; */
|
||||
/* } */
|
||||
|
||||
|
||||
/* Execute the desired action(s) */
|
||||
/* /\* Execute the desired action(s) *\/ */
|
||||
|
||||
if (ompi_cmd_line_is_taken(ompi_common_cmd_line, "version")) {
|
||||
printf ("ompid (OpenMpi Daemon) version: 0\n");
|
||||
}
|
||||
|
||||
/* If there is a seed argument, this is the magic
|
||||
* seed daemon.
|
||||
*/
|
||||
if ( ompi_cmd_line_is_taken(cmd_line, "seed")) {
|
||||
ompi_process_info.seed = true;
|
||||
/* create session directory */
|
||||
}
|
||||
/* if (ompi_cmd_line_is_taken(ompi_common_cmd_line, "version")) { */
|
||||
/* printf ("ompid (OpenMpi Daemon) version: 0\n"); */
|
||||
/* } */
|
||||
|
||||
|
||||
/* before calling anything we to call rte init */
|
||||
if (OMPI_SUCCESS != ompi_rte_init(&multi_thread, &hidden_thread)) {
|
||||
printf("ompid: ompi_rte_init failed\n");
|
||||
/* /\* setup universe session directory *\/ */
|
||||
/* if (OMPI_SUCCESS != ompi_session_dir(true, tmpdir, ompi_system_info.user, ompi_system_info.nodename, NULL, */
|
||||
/* ompi_universe.name, NULL, NULL)) { /\* couldn't create session dir - error *\/ */
|
||||
/* fprintf(stderr, "could not create universe session directory tree - please report error to bugs@open-mpi.org\n"); */
|
||||
/* exit(1); */
|
||||
/* } */
|
||||
|
||||
/* Do a partial clean-up. This needs to be reviewed
|
||||
* at a later date to make certain we are not
|
||||
* missing soemthing
|
||||
*/
|
||||
ompi_rte_finalize();
|
||||
OBJ_RELEASE(cmd_line);
|
||||
mca_base_close();
|
||||
/* /\* If there is a seed argument, this is the magic */
|
||||
/* * seed daemon. */
|
||||
/* *\/ */
|
||||
/* if ( ompi_cmd_line_is_taken(cmd_line, "seed")) { */
|
||||
/* ompi_process_info.seed = true; */
|
||||
/* ompi_process_info.my_universe = strdup(ompi_universe.name); */
|
||||
/* } */
|
||||
|
||||
return 1;
|
||||
}
|
||||
/* /\* convert myself to be a daemon *\/ */
|
||||
/* if (OMPI_SUCCESS != ompi_daemon_init(ompi_process_info.universe_session_dir)) { */
|
||||
/* fprintf(stderr, "could not convert to daemon - please report error to bugs@open-mpi.org\n"); */
|
||||
/* exit(1); */
|
||||
/* } */
|
||||
|
||||
|
||||
/*
|
||||
* if seed, call open functions of comm frameworks (oob, socket, etc.) to
|
||||
* get contact info. write contact info into universe session directory
|
||||
* as file "contact-info" so others can find us.
|
||||
*/
|
||||
/* /\* before calling anything else we to call rte init *\/ */
|
||||
/* if (OMPI_SUCCESS != ompi_rte_init(&multi_thread, &hidden_thread)) { */
|
||||
/* printf("ompid: ompi_rte_init failed\n"); */
|
||||
|
||||
/* Add in the calls to initialize the services */
|
||||
/* /\* Do a partial clean-up. This needs to be reviewed */
|
||||
/* * at a later date to make certain we are not */
|
||||
/* * missing soemthing */
|
||||
/* *\/ */
|
||||
/* ompi_rte_finalize(); */
|
||||
/* OBJ_RELEASE(cmd_line); */
|
||||
/* mca_base_close(); */
|
||||
|
||||
/* Add the section for the event loop... */
|
||||
/* return 1; */
|
||||
/* } */
|
||||
|
||||
/* All done */
|
||||
|
||||
/* Close services */
|
||||
/* /\* */
|
||||
/* * if seed, call open functions of comm frameworks (oob, socket, etc.) to */
|
||||
/* * get contact info. write contact info into universe session directory */
|
||||
/* * as file "contact-info" so others can find us. */
|
||||
/* *\/ */
|
||||
|
||||
OBJ_RELEASE(cmd_line);
|
||||
mca_base_close();
|
||||
ompi_finalize();
|
||||
return 0;
|
||||
/* /\* Add in the calls to initialize the services *\/ */
|
||||
|
||||
/* /\* Add the section for the event loop... *\/ */
|
||||
|
||||
/* /\* All done *\/ */
|
||||
|
||||
/* /\* Close services *\/ */
|
||||
|
||||
/* OBJ_RELEASE(cmd_line); */
|
||||
/* mca_base_close(); */
|
||||
/* ompi_finalize(); */
|
||||
/* return 0; */
|
||||
}
|
||||
|
@ -13,6 +13,8 @@
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <unistd.h>
|
||||
#include <sys/types.h>
|
||||
|
||||
#include "runtime/runtime.h"
|
||||
#include "runtime/universe_connect.h"
|
||||
@ -29,22 +31,9 @@
|
||||
#include "util/universe_setup_file_io.h"
|
||||
#include "mca/base/base.h"
|
||||
#include "mca/oob/base/base.h"
|
||||
#include "tools/openmpi/openmpi.h"
|
||||
|
||||
|
||||
ompi_universe_t ompi_universe = {
|
||||
/* .name = */ "default-universe",
|
||||
/* .host = */ NULL,
|
||||
/* .uid = */ NULL,
|
||||
/* .persistence = */ false,
|
||||
/* .scope = */ "local",
|
||||
/* .silent_mode = */ false,
|
||||
/* .script_mode = */ false,
|
||||
/* .web_server = */ false,
|
||||
/* .socket_contact_info = */ NULL,
|
||||
/* .oob_contact_info = */ NULL,
|
||||
/* .console_connected = */ false
|
||||
};
|
||||
extern ompi_universe_t ompi_universe_info;
|
||||
|
||||
|
||||
int main(int argc, char **argv)
|
||||
@ -56,6 +45,7 @@ int main(int argc, char **argv)
|
||||
char *script_file;
|
||||
char *contact_file;
|
||||
int ret;
|
||||
pid_t pid;
|
||||
bool persistent, silent, script, webserver;
|
||||
bool multi_thread = false;
|
||||
bool hidden_thread = false;
|
||||
@ -77,9 +67,8 @@ int main(int argc, char **argv)
|
||||
|
||||
/* get the system info and setup defaults */
|
||||
ompi_sys_info();
|
||||
ompi_universe.host = strdup(ompi_system_info.nodename);
|
||||
ompi_universe.uid = strdup(ompi_system_info.user);
|
||||
ompi_universe.name = strdup("default-universe");
|
||||
ompi_universe_info.host = strdup(ompi_system_info.nodename);
|
||||
ompi_universe_info.uid = strdup(ompi_system_info.user);
|
||||
|
||||
|
||||
/* setup to read common command line options that span all Open MPI programs */
|
||||
@ -93,215 +82,119 @@ int main(int argc, char **argv)
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* setup the rte command line arguments */
|
||||
cmd_line = OBJ_NEW(ompi_cmd_line_t);
|
||||
ompi_cmd_line_make_opt(cmd_line, 's', "seed", 0,
|
||||
"Set the daemon seed to true.");
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line,
|
||||
'u', "universe", 1,
|
||||
"Specify the Open MPI universe");
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line,
|
||||
't', "tmpdir", 1,
|
||||
"Specify the Open MPI prefix for the session directory");
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line, 'w', "webserver", 0,
|
||||
"Web server available");
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line, 's', "silent", 0,
|
||||
"No console prompt - operate silently");
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line, 'f', "script", 1,
|
||||
"Read commands from script file");
|
||||
|
||||
ompi_cmd_line_make_opt(cmd_line, 'c', "scope", 1,
|
||||
"Scope of this universe");
|
||||
|
||||
/*
|
||||
* setup mca command line arguments
|
||||
*/
|
||||
mca_cmd_line = OBJ_NEW(ompi_cmd_line_t);
|
||||
if (OMPI_SUCCESS != (ret = mca_base_cmd_line_setup(mca_cmd_line))) {
|
||||
/* BWB show_help */
|
||||
printf("show_help: mca_base_cmd_line_setup failed\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (OMPI_SUCCESS != mca_base_cmd_line_process_args(mca_cmd_line)) {
|
||||
/* BWB show_help */
|
||||
printf("show_help: mca_base_cmd_line_process_args\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* parse the local commands */
|
||||
if (OMPI_SUCCESS != ompi_cmd_line_parse(cmd_line, true, argc, argv)) {
|
||||
exit(ret);
|
||||
}
|
||||
|
||||
/* get universe name and store it, if user specified it */
|
||||
/* otherwise, stick with default name */
|
||||
if (ompi_cmd_line_is_taken(cmd_line, "universe")) {
|
||||
if (NULL == ompi_cmd_line_get_param(cmd_line, "universe", 0, 0)) {
|
||||
fprintf(stderr, "error retrieving universe name - please report error to bugs@open-mpi.org\n");
|
||||
exit(1);
|
||||
}
|
||||
universe = strdup(ompi_cmd_line_get_param(cmd_line, "universe", 0, 0));
|
||||
|
||||
if (NULL != (tmp = strchr(universe, ':'))) { /* name contains remote host */
|
||||
/* get the host name, and the universe name separated */
|
||||
/* could be in form remote-uid@remote-host:universe */
|
||||
*tmp = '\0';
|
||||
tmp++;
|
||||
ompi_universe.name = strdup(tmp);
|
||||
if (NULL != (tmp = strchr(universe, '@'))) { /* remote name includes remote uid */
|
||||
*tmp = '\0';
|
||||
tmp++;
|
||||
ompi_universe.host = strdup(tmp);
|
||||
ompi_universe.uid = strdup(universe);
|
||||
} else { /* no remote id - just remote host */
|
||||
ompi_universe.host = strdup(universe);
|
||||
}
|
||||
} else { /* no remote host - just universe name provided */
|
||||
ompi_universe.name = strdup(universe);
|
||||
}
|
||||
}
|
||||
|
||||
/* get desired universe scope, if specified */
|
||||
if (ompi_cmd_line_is_taken(cmd_line, "scope")) {
|
||||
}
|
||||
/* get the temporary directory name for the session directory, if provided on command line */
|
||||
if (ompi_cmd_line_is_taken(cmd_line, "tmpdir")) {
|
||||
if (NULL == ompi_cmd_line_get_param(cmd_line, "tmpdir", 0, 0)) {
|
||||
fprintf(stderr, "error retrieving tmpdir name - please report error to bugs@open-mpi.org\n");
|
||||
exit(1);
|
||||
}
|
||||
tmpdir = strdup(ompi_cmd_line_get_param(cmd_line, "tmpdir", 0, 0));
|
||||
} else {
|
||||
tmpdir = NULL;
|
||||
}
|
||||
|
||||
/* find out if silent */
|
||||
if (ompi_cmd_line_is_taken(cmd_line, "silent")) {
|
||||
silent = true;
|
||||
}
|
||||
|
||||
/* find out if web interface is desired */
|
||||
if (ompi_cmd_line_is_taken(cmd_line, "webserver")) {
|
||||
webserver = true;
|
||||
}
|
||||
|
||||
/* find out if script is to be executed */
|
||||
if (ompi_cmd_line_is_taken(cmd_line, "script")) {
|
||||
script = true;
|
||||
if (NULL == ompi_cmd_line_get_param(cmd_line, "script", 0, 0)) {
|
||||
fprintf(stderr, "error retrieving script file name - please report error to bugs@open-mpi.org\n");
|
||||
exit(1);
|
||||
}
|
||||
script_file = strdup(ompi_cmd_line_get_param(cmd_line, "script", 0, 0));
|
||||
}
|
||||
|
||||
/* does universe already exist on specified host? Check session directory to see */
|
||||
/* don't know how to handle remote host yet - only cover localhost */
|
||||
|
||||
if (0 == strncmp(ompi_universe.host, ompi_system_info.nodename, strlen(ompi_system_info.nodename))) { /* localhost specified or defaulted */
|
||||
if (OMPI_SUCCESS == ompi_session_dir(false, tmpdir, ompi_system_info.user, ompi_system_info.nodename, NULL,
|
||||
ompi_universe.name, NULL, NULL)) { /* found */
|
||||
fprintf(stderr, "think i found something\n");
|
||||
/* check for "contact-info" file. if present, read it in. if not present, wait one second (might
|
||||
* be race condition) and try again.
|
||||
*/
|
||||
if (OMPI_SUCCESS != ompi_session_dir(true, tmpdir, ompi_system_info.user, ompi_system_info.nodename, NULL,
|
||||
ompi_universe.name, NULL, NULL)) {
|
||||
fprintf(stderr, "couldn't update the process info structure - please report error to bugs@open-mpi.org\n");
|
||||
exit(1);
|
||||
}
|
||||
contact_file = ompi_os_path(false, ompi_process_info.universe_session_dir,
|
||||
"universe-setup.txt", NULL);
|
||||
|
||||
if (OMPI_SUCCESS != (ret = ompi_read_universe_setup_file(contact_file, &ompi_universe))) {
|
||||
if (OMPI_ERR_NOT_FOUND == ret) { /* couldn't find file - assume prior seed daemon died */
|
||||
goto STARTUP;
|
||||
} else {
|
||||
fprintf(stderr, "couldn't read contact info: %s\n", contact_file);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (!ompi_universe.persistence) { /* if not persistent, define our own name and start new universe */
|
||||
/* derive unique name based on current one */
|
||||
goto STARTUP;
|
||||
}
|
||||
|
||||
/* if persistent, use contact info to connect */
|
||||
|
||||
if (OMPI_ERROR == ompi_universe_connect(ompi_universe.oob_contact_info)) { /* try to connect */
|
||||
/* universe must have died - try starting up new one */
|
||||
goto STARTUP;
|
||||
}
|
||||
|
||||
} else {
|
||||
fprintf(stderr, "session dir not found - creating it - calling univ_init\n");
|
||||
/* setup universe session directory */
|
||||
if (OMPI_SUCCESS != ompi_session_dir(true, tmpdir, ompi_system_info.user, ompi_system_info.nodename, NULL,
|
||||
ompi_universe.name, NULL, NULL)) { /* couldn't create session dir - error */
|
||||
fprintf(stderr, "could not create universe session directory tree - please report error to bugs@open-mpi.org\n");
|
||||
if (ompi_cmd_line_is_taken(ompi_common_cmd_line, "version") ||
|
||||
ompi_cmd_line_is_taken(ompi_common_cmd_line, "v")) {
|
||||
printf("...showing off my version!\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* convert myself to be the seed daemon */
|
||||
STARTUP:
|
||||
ompi_process_info.seed = true;
|
||||
ompi_process_info.my_universe = strdup(ompi_universe.name);
|
||||
/* /\* setup rte command line arguments *\/ */
|
||||
/* cmd_line = OBJ_NEW(ompi_cmd_line_t); */
|
||||
/* ompi_rte_cmd_line_setup(cmd_line); */
|
||||
|
||||
#if 0
|
||||
if (OMPI_SUCCESS != ompi_daemon_init(ompi_process_info.universe_session_dir)) {
|
||||
fprintf(stderr, "could not convert to daemon - please report error to bugs@open-mpi.org\n");
|
||||
exit(1);
|
||||
}
|
||||
#endif
|
||||
/* /\* */
|
||||
/* * setup mca command line arguments */
|
||||
/* *\/ */
|
||||
/* mca_cmd_line = OBJ_NEW(ompi_cmd_line_t); */
|
||||
/* if (OMPI_SUCCESS != (ret = mca_base_cmd_line_setup(mca_cmd_line))) { */
|
||||
/* /\* BWB show_help *\/ */
|
||||
/* printf("show_help: mca_base_cmd_line_setup failed\n"); */
|
||||
/* return ret; */
|
||||
/* } */
|
||||
|
||||
/*
|
||||
* Start the Open MPI Run Time Environment
|
||||
*/
|
||||
if (OMPI_SUCCESS != (ret = mca_base_open())) {
|
||||
/* JMS show_help */
|
||||
printf("show_help: mca_base_open failed\n");
|
||||
return ret;
|
||||
}
|
||||
/* if (OMPI_SUCCESS != mca_base_cmd_line_process_args(mca_cmd_line)) { */
|
||||
/* /\* BWB show_help *\/ */
|
||||
/* printf("show_help: mca_base_cmd_line_process_args\n"); */
|
||||
/* return ret; */
|
||||
/* } */
|
||||
|
||||
if (OMPI_SUCCESS != ompi_rte_init(&multi_thread, &hidden_thread)) {
|
||||
/* BWB show_help */
|
||||
printf("show_help: ompi_rte_init failed\n");
|
||||
return ret;
|
||||
}
|
||||
/* /\* parse the local commands *\/ */
|
||||
/* if (OMPI_SUCCESS != ompi_cmd_line_parse(cmd_line, true, argc, argv)) { */
|
||||
/* exit(ret); */
|
||||
/* } */
|
||||
|
||||
/* get OOB contact info */
|
||||
ompi_universe.oob_contact_info = mca_oob_get_contact_info();
|
||||
|
||||
/* get Web contact info */
|
||||
ompi_universe.socket_contact_info = strdup("dum.add.for.tst");
|
||||
/* /\* parse the cmd_line for seed daemon options *\/ */
|
||||
/* ompi_rte_parse_seed_cmd_line(cmd_line); */
|
||||
|
||||
/* save all pertinent info in universe file */
|
||||
contact_file = ompi_os_path(false, ompi_process_info.universe_session_dir,
|
||||
"universe-setup.txt", NULL);
|
||||
/* /\* check for universe existence *\/ */
|
||||
/* if (OMPI_SUCCESS != ompi_rte_universe_exists()) { */
|
||||
/* /\* check exit codes */
|
||||
/* * exists, but connect not allowed or refused *\/ */
|
||||
/* /\*define unique name based on current one and start new universe *\/ */
|
||||
/* pid = getpid(); */
|
||||
/* if (0 < asprintf(&ompi_universe.name, "%s-%d", universe, pid)) { */
|
||||
/* fprintf(stderr, "error creating unique universe name - please report error to bugs@open-mpi.org\n"); */
|
||||
/* exit(1); */
|
||||
/* } */
|
||||
|
||||
if (OMPI_SUCCESS != ompi_write_universe_setup_file(contact_file, &ompi_universe)) {
|
||||
fprintf(stderr, "couldn't write universe setup file: %s\n", contact_file);
|
||||
exit(1);
|
||||
}
|
||||
/* /\* does not exist - need to start *\/ */
|
||||
/* if (OMPI_SUCCESS != ompi_rte_universe_initiate()) { */
|
||||
/* fprintf(stderr, "unable to start universe services - please report error to bugs@open-mpi.org\n"); */
|
||||
/* exit(1); */
|
||||
/* } */
|
||||
/* } */
|
||||
|
||||
/* put info on the registry */
|
||||
/* /\* universe already exists and we can connect - see what else needs doing *\/ */
|
||||
/* goto OPERATE; */
|
||||
|
||||
fprintf(stderr, "openmpi: entering event loop\n");
|
||||
/* event loop */
|
||||
ompi_event_loop(0);
|
||||
}
|
||||
}
|
||||
/* spawn console process */
|
||||
if (!silent) {
|
||||
fprintf(stderr, "SUCCESS - spawned console process!\n");
|
||||
}
|
||||
/* } */
|
||||
|
||||
return(0);
|
||||
/* /\* universe must not already exist, so create it from scratch *\/ */
|
||||
/* STARTUP: */
|
||||
|
||||
/* /\* spinoff ompid with "seed" flag set - it will become the seed daemon for this universe *\/ */
|
||||
|
||||
/* /\* */
|
||||
/* * Start the Open MPI Run Time Environment */
|
||||
/* *\/ */
|
||||
/* if (OMPI_SUCCESS != (ret = mca_base_open())) { */
|
||||
/* /\* JMS show_help *\/ */
|
||||
/* printf("show_help: mca_base_open failed\n"); */
|
||||
/* return ret; */
|
||||
/* } */
|
||||
|
||||
/* if (OMPI_SUCCESS != ompi_rte_init(&multi_thread, &hidden_thread)) { */
|
||||
/* /\* BWB show_help *\/ */
|
||||
/* printf("show_help: ompi_rte_init failed\n"); */
|
||||
/* return ret; */
|
||||
/* } */
|
||||
|
||||
/* /\* get OOB contact info *\/ */
|
||||
/* ompi_universe.oob_contact_info = mca_oob_get_contact_info(); */
|
||||
|
||||
/* /\* get Web contact info *\/ */
|
||||
/* ompi_universe.socket_contact_info = strdup("dum.add.for.tst"); */
|
||||
|
||||
/* /\* save all pertinent info in universe file *\/ */
|
||||
/* contact_file = ompi_os_path(false, ompi_process_info.universe_session_dir, */
|
||||
/* "universe-setup.txt", NULL); */
|
||||
|
||||
/* if (OMPI_SUCCESS != ompi_write_universe_setup_file(contact_file, &ompi_universe)) { */
|
||||
/* fprintf(stderr, "couldn't write universe setup file: %s\n", contact_file); */
|
||||
/* exit(1); */
|
||||
/* } */
|
||||
|
||||
/* /\* put info on the registry *\/ */
|
||||
|
||||
/* fprintf(stderr, "openmpi: entering event loop\n"); */
|
||||
/* /\* event loop *\/ */
|
||||
/* ompi_event_loop(0); */
|
||||
/* /\* spawn console process *\/ */
|
||||
/* if (!silent) { */
|
||||
/* fprintf(stderr, "SUCCESS - spawned console process!\n"); */
|
||||
/* } */
|
||||
|
||||
/* OPERATE: */
|
||||
/* /\* if hostfile, startup virtual machine *\/ */
|
||||
/* /\* check registry for nodes in hostfile - if not found, add them *\/ */
|
||||
/* /\* send command - ompi_vm_startup to seed that causes it to read registry segment, check if ompid already */
|
||||
/* * on each node, spin one up if not *\/ */
|
||||
|
||||
/* /\* if console, kickoff console - point comm at universe *\/ */
|
||||
|
||||
/* /\* all done, so exit! *\/ */
|
||||
/* exit(0); */
|
||||
|
||||
exit(0);
|
||||
}
|
||||
|
@ -8,29 +8,6 @@
|
||||
#ifndef OMPI_UNIVERSE_H
|
||||
#define OMPI_UNIVERSE_H
|
||||
|
||||
/* Define the structures underlying the Open MPI universe system */
|
||||
|
||||
struct ompi_universe_t {
|
||||
char *name;
|
||||
char *host;
|
||||
char *uid;
|
||||
bool persistence;
|
||||
char *scope;
|
||||
bool silent_mode;
|
||||
bool script_mode;
|
||||
bool web_server;
|
||||
char *socket_contact_info;
|
||||
char *oob_contact_info;
|
||||
bool console_connected;
|
||||
};
|
||||
typedef struct ompi_universe_t ompi_universe_t;
|
||||
|
||||
extern ompi_universe_t ompi_universe;
|
||||
|
||||
#ifndef MAXHOSTNAMELEN
|
||||
#define MAXHOSTNAMELEN 256
|
||||
#endif
|
||||
|
||||
#define OMPI_RIDICULOUS_NAMELEN 1024
|
||||
|
||||
#endif
|
||||
|
@ -22,7 +22,18 @@ ompi_proc_info_t ompi_process_info = {
|
||||
/* .pid = */ 0,
|
||||
/* .name = */ NULL,
|
||||
/* .seed = */ false,
|
||||
/* .my_universe = */ "default-universe",
|
||||
/* .my_universe = name */ {"default-universe",
|
||||
/* host */ "localhost",
|
||||
/* uid */ NULL,
|
||||
/* persistence */ false,
|
||||
/* scope */ "local",
|
||||
/* silent */ false,
|
||||
/* web_server */ false,
|
||||
/* scriptfile */ NULL,
|
||||
/* hostfile */ NULL,
|
||||
/* socket_contact_info */ NULL,
|
||||
/* oob_contact_info */ NULL,
|
||||
/* console_connected */ false},
|
||||
/* .tmpdir_base = */ NULL,
|
||||
/* .top_session_dir = */ NULL,
|
||||
/* .universe_session_dir = */ NULL,
|
||||
|
@ -13,6 +13,7 @@
|
||||
#include <sys/types.h>
|
||||
#include "mca/ns/ns.h"
|
||||
|
||||
|
||||
/**
|
||||
* Process information structure
|
||||
*
|
||||
@ -30,7 +31,7 @@ struct ompi_proc_info_t {
|
||||
pid_t pid; /**< Local process ID for this process */
|
||||
ompi_process_name_t *name; /**< Process name structure */
|
||||
bool seed; /**< Indicate whether or not this is seed daemon */
|
||||
char *my_universe; /**< The name of the universe to which this process belongs */
|
||||
char *my_universe; /**< Name of the universe to which this process belongs */
|
||||
char *tmpdir_base; /**< Base directory of the session dir tree */
|
||||
char *top_session_dir; /**< Top-most directory of the session tree */
|
||||
char *universe_session_dir; /**< Location of universe temp dir.
|
||||
|
@ -16,14 +16,14 @@
|
||||
#include "include/constants.h"
|
||||
|
||||
#include "util/output.h"
|
||||
#include "tools/openmpi/openmpi.h"
|
||||
#include "runtime/runtime.h"
|
||||
#include "util/universe_setup_file_io.h"
|
||||
|
||||
#define OMPI_UNIV_SETUP_FILE_MAX_LINE_LENGTH 1024
|
||||
|
||||
char *ompi_getline(FILE *fp);
|
||||
|
||||
int ompi_write_universe_setup_file(char *filename, ompi_universe_t *universe)
|
||||
int ompi_write_universe_setup_file(char *filename)
|
||||
{
|
||||
FILE *fp;
|
||||
|
||||
@ -32,48 +32,48 @@ int ompi_write_universe_setup_file(char *filename, ompi_universe_t *universe)
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
if (NULL == universe->name) {
|
||||
if (NULL == ompi_universe_info.name) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
fprintf(fp, "%s\n", universe->name);
|
||||
fprintf(fp, "%s\n", ompi_universe_info.name);
|
||||
|
||||
if (NULL == universe->host) {
|
||||
if (NULL == ompi_universe_info.host) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
fprintf(fp, "%s\n", universe->host);
|
||||
fprintf(fp, "%s\n", ompi_universe_info.host);
|
||||
|
||||
if (NULL == universe->uid) {
|
||||
if (NULL == ompi_universe_info.uid) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
fprintf(fp, "%s\n", universe->uid);
|
||||
fprintf(fp, "%s\n", ompi_universe_info.uid);
|
||||
|
||||
if (universe->persistence) {
|
||||
if (ompi_universe_info.persistence) {
|
||||
fprintf(fp, "persistent\n");
|
||||
} else {
|
||||
fprintf(fp, "non-persistent\n");
|
||||
}
|
||||
|
||||
if (NULL == universe->scope) {
|
||||
if (NULL == ompi_universe_info.scope) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
fprintf(fp, "%s\n", universe->scope);
|
||||
fprintf(fp, "%s\n", ompi_universe_info.scope);
|
||||
|
||||
if (universe->silent_mode) {
|
||||
if (ompi_universe_info.silent_mode) {
|
||||
fprintf(fp, "silent\n");
|
||||
} else {
|
||||
fprintf(fp, "console\n");
|
||||
}
|
||||
|
||||
if (universe->web_server && NULL != universe->socket_contact_info) {
|
||||
fprintf(fp, "%s\n", universe->socket_contact_info);
|
||||
if (ompi_universe_info.web_server && NULL != ompi_universe_info.socket_contact_info) {
|
||||
fprintf(fp, "%s\n", ompi_universe_info.socket_contact_info);
|
||||
} else {
|
||||
fprintf(fp, "none\n");
|
||||
}
|
||||
|
||||
if (NULL == universe->oob_contact_info) {
|
||||
if (NULL == ompi_universe_info.oob_contact_info) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
fprintf(fp, "%s\n", universe->oob_contact_info);
|
||||
fprintf(fp, "%s\n", ompi_universe_info.oob_contact_info);
|
||||
fclose(fp);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
@ -83,7 +83,7 @@ int ompi_write_universe_setup_file(char *filename, ompi_universe_t *universe)
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
int ompi_read_universe_setup_file(char *filename, ompi_universe_t *universe)
|
||||
int ompi_read_universe_setup_file(char *filename)
|
||||
{
|
||||
char *input;
|
||||
FILE *fp;
|
||||
@ -97,18 +97,18 @@ int ompi_read_universe_setup_file(char *filename, ompi_universe_t *universe)
|
||||
}
|
||||
}
|
||||
|
||||
universe->name = ompi_getline(fp);
|
||||
if (NULL == universe->name) {
|
||||
ompi_universe_info.name = ompi_getline(fp);
|
||||
if (NULL == ompi_universe_info.name) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
|
||||
universe->host = ompi_getline(fp);
|
||||
if (NULL == universe->host) {
|
||||
ompi_universe_info.host = ompi_getline(fp);
|
||||
if (NULL == ompi_universe_info.host) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
|
||||
universe->uid = ompi_getline(fp);
|
||||
if (NULL == universe->uid) {
|
||||
ompi_universe_info.uid = ompi_getline(fp);
|
||||
if (NULL == ompi_universe_info.uid) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
|
||||
@ -117,17 +117,17 @@ int ompi_read_universe_setup_file(char *filename, ompi_universe_t *universe)
|
||||
goto CLEANUP;
|
||||
}
|
||||
if (0 == strncmp(input, "persistent", strlen("persistent"))) {
|
||||
universe->persistence = true;
|
||||
ompi_universe_info.persistence = true;
|
||||
} else if (0 == strncmp(input, "non-persistent", strlen("non-persistent"))) {
|
||||
universe->persistence = false;
|
||||
ompi_universe_info.persistence = false;
|
||||
} else {
|
||||
free(input);
|
||||
goto CLEANUP;
|
||||
}
|
||||
free(input);
|
||||
|
||||
universe->scope = ompi_getline(fp);
|
||||
if (NULL == universe->scope) {
|
||||
ompi_universe_info.scope = ompi_getline(fp);
|
||||
if (NULL == ompi_universe_info.scope) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
|
||||
@ -136,27 +136,27 @@ int ompi_read_universe_setup_file(char *filename, ompi_universe_t *universe)
|
||||
goto CLEANUP;
|
||||
}
|
||||
if (0 == strncmp(input, "silent", strlen("silent"))) {
|
||||
universe->silent_mode = true;
|
||||
ompi_universe_info.silent_mode = true;
|
||||
} else if (0 == strncmp(input, "console", strlen("console"))) {
|
||||
universe->silent_mode = false;
|
||||
ompi_universe_info.silent_mode = false;
|
||||
} else {
|
||||
free(input);
|
||||
goto CLEANUP;
|
||||
}
|
||||
free(input);
|
||||
|
||||
universe->socket_contact_info = ompi_getline(fp);
|
||||
if (NULL == universe->socket_contact_info) {
|
||||
ompi_universe_info.socket_contact_info = ompi_getline(fp);
|
||||
if (NULL == ompi_universe_info.socket_contact_info) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
if (0 == strncmp(universe->socket_contact_info, "none", strlen("none"))) {
|
||||
universe->web_server = false;
|
||||
if (0 == strncmp(ompi_universe_info.socket_contact_info, "none", strlen("none"))) {
|
||||
ompi_universe_info.web_server = false;
|
||||
} else {
|
||||
universe->web_server = true;
|
||||
ompi_universe_info.web_server = true;
|
||||
}
|
||||
|
||||
universe->oob_contact_info = ompi_getline(fp);
|
||||
if (NULL == universe->oob_contact_info) {
|
||||
ompi_universe_info.oob_contact_info = ompi_getline(fp);
|
||||
if (NULL == ompi_universe_info.oob_contact_info) {
|
||||
goto CLEANUP;
|
||||
}
|
||||
|
||||
|
@ -11,10 +11,8 @@
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "tools/openmpi/openmpi.h"
|
||||
int ompi_write_universe_setup_file(char *filename);
|
||||
|
||||
int ompi_write_universe_setup_file(char *filename, ompi_universe_t *universe);
|
||||
|
||||
int ompi_read_universe_setup_file(char *filename, ompi_universe_t *universe);
|
||||
int ompi_read_universe_setup_file(char *filename);
|
||||
|
||||
#endif
|
||||
|
Загрузка…
x
Ссылка в новой задаче
Block a user