ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Cleanup s1 and s2 components, and ensure that mpirun and orteds only use non-direct-launch pmix components.

Просмотр исходного кода
Этот коммит содержится в:
Ralph Castain 2015-09-08 18:37:09 -07:00
родитель bc9e5652ff
Коммит 1cdb86b8c7
6 изменённых файлов: 133 добавлений и 131 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

14
opal/mca/pmix/base/pmix_base_hash.c
Просмотреть файл

@ -149,14 +149,14 @@ int opal_pmix_base_store(const opal_process_name_t *id,
int rc;
opal_output_verbose(1, opal_pmix_base_framework.framework_output,
"%s dstore:hash:store storing data for proc %s",
"%s pmix:hash:store storing data for proc %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), OPAL_NAME_PRINT(*id));
/* lookup the proc data object for this proc */
if (NULL == (proc_data = lookup_proc(&ptable, *id, true))) {
/* unrecoverable error */
OPAL_OUTPUT_VERBOSE((5, opal_pmix_base_framework.framework_output,
"%s dstore:hash:store: storing data for proc %s unrecoverably failed",
"%s pmix:hash:store: storing data for proc %s unrecoverably failed",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME), OPAL_NAME_PRINT(*id)));
return OPAL_ERR_OUT_OF_RESOURCE;
}
@ -168,7 +168,7 @@ int opal_pmix_base_store(const opal_process_name_t *id,
#if OPAL_ENABLE_DEBUG
char *_data_type = opal_dss.lookup_data_type(val->type);
OPAL_OUTPUT_VERBOSE((5, opal_pmix_base_framework.framework_output,
"%s dstore:hash:store: %s key %s[%s] for proc %s",
"%s pmix:hash:store: %s key %s[%s] for proc %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
(NULL == kv ? "storing" : "updating"),
val->key, _data_type, OPAL_NAME_PRINT(*id)));
@ -197,14 +197,14 @@ int opal_pmix_base_fetch(const opal_process_name_t *id,
int rc;
OPAL_OUTPUT_VERBOSE((5, opal_pmix_base_framework.framework_output,
"%s dstore:hash:fetch: searching for key %s on proc %s",
"%s pmix:hash:fetch: searching for key %s on proc %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
(NULL == key) ? "NULL" : key, OPAL_NAME_PRINT(*id)));
/* lookup the proc data object for this proc */
if (NULL == (proc_data = lookup_proc(&ptable, *id, true))) {
OPAL_OUTPUT_VERBOSE((5, opal_pmix_base_framework.framework_output,
"%s dstore_hash:fetch data for proc %s not found",
"%s pmix_hash:fetch data for proc %s not found",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
OPAL_NAME_PRINT(*id)));
return OPAL_ERR_NOT_FOUND;
@ -224,7 +224,7 @@ int opal_pmix_base_fetch(const opal_process_name_t *id,
return rc;
}
OPAL_OUTPUT_VERBOSE((5, opal_pmix_base_framework.framework_output,
"%s dstore:hash:fetch: adding data for key %s on proc %s",
"%s pmix:hash:fetch: adding data for key %s on proc %s",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
(NULL == kv->key) ? "NULL" : kv->key,
OPAL_NAME_PRINT(*id)));
@ -238,7 +238,7 @@ int opal_pmix_base_fetch(const opal_process_name_t *id,
/* find the value */
if (NULL == (kv = lookup_keyval(proc_data, key))) {
OPAL_OUTPUT_VERBOSE((5, opal_pmix_base_framework.framework_output,
"%s dstore_hash:fetch key %s for proc %s not found",
"%s pmix_hash:fetch key %s for proc %s not found",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
(NULL == key) ? "NULL" : key,
OPAL_NAME_PRINT(*id)));

85
opal/mca/pmix/s1/pmix_s1.c
Просмотреть файл

@ -41,12 +41,9 @@ static int s1_put(opal_pmix_scope_t scope,
static int s1_get(const opal_process_name_t *id,
const char *key,
opal_value_t **kv);
static int s1_publish(opal_pmix_data_range_t scope,
opal_pmix_persistence_t persist,
opal_list_t *info);
static int s1_lookup(opal_pmix_data_range_t scope,
opal_list_t *data);
static int s1_unpublish(opal_pmix_data_range_t scope, char **keys);
static int s1_publish(opal_list_t *info);
static int s1_lookup(opal_list_t *data, opal_list_t *info);
static int s1_unpublish(char **keys, opal_list_t *info);
static int s1_spawn(opal_list_t *jobinfo, opal_list_t *apps, opal_jobid_t *jobid);
static int s1_job_connect(opal_list_t *procs);
static int s1_job_disconnect(opal_list_t *procs);
@ -164,7 +161,7 @@ static int s1_init(void)
return OPAL_ERROR;
}
if( PMI_TRUE != initialized && PMI_SUCCESS != (rc = PMI_Init(&spawned)) ) {
if (PMI_TRUE != initialized && PMI_SUCCESS != (rc = PMI_Init(&spawned))) {
OPAL_PMI_ERROR(rc, "PMI_Init");
return OPAL_ERROR;
}
@ -174,7 +171,7 @@ static int s1_init(void)
// Initialize space demands
rc = PMI_KVS_Get_value_length_max(&pmix_vallen_max);
if( PMI_SUCCESS != rc ) {
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_value_length_max");
goto err_exit;
}
@ -182,20 +179,20 @@ static int s1_init(void)
pmix_vallen_threshold >>= 2;
rc = PMI_KVS_Get_name_length_max(&pmix_kvslen_max);
if (PMI_SUCCESS != rc ) {
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_name_length_max");
goto err_exit;
}
rc = PMI_KVS_Get_key_length_max(&pmix_keylen_max);
if( PMI_SUCCESS != rc ) {
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_key_length_max");
goto err_exit;
}
// Initialize job environment information
pmix_id = (char*)malloc(pmix_vallen_max);
if( pmix_id == NULL ){
if (pmix_id == NULL) {
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
@ -204,11 +201,36 @@ static int s1_init(void)
free(pmix_id);
goto err_exit;
}
/* get our rank */
ret = PMI_Get_rank(&rank);
if( PMI_SUCCESS != ret ) {
OPAL_PMI_ERROR(ret, "PMI_Get_rank");
goto err_exit;
}
/* Slurm PMI provides the job id as an integer followed
* by a '.', followed by essentially a stepid. The first integer
* defines an overall job number. The second integer is the number of
* individual jobs we have run within that allocation.
*/
* individual jobs we have run within that allocation. */
s1_pname.jobid = strtoul(pmix_id, &str, 10);
s1_pname.jobid = (s1_pname.jobid << 16) & 0xffff0000;
if (NULL != str) {
ui32 = strtoul(str, NULL, 10);
s1_pname.jobid |= (ui32 & 0x0000ffff);
}
free(pmix_id);
ldr.jobid = s1_pname.jobid;
s1_pname.vpid = rank;
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
opal_proc_set_name(&s1_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s1: assigned tmp name",
OPAL_NAME_PRINT(s1_pname));
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_STRING;
@ -220,12 +242,6 @@ static int s1_init(void)
}
OBJ_DESTRUCT(&kv);
/* get our rank */
ret = PMI_Get_rank(&rank);
if( PMI_SUCCESS != ret ) {
OPAL_PMI_ERROR(ret, "PMI_Get_rank");
goto err_exit;
}
/* save it */
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_RANK);
@ -238,32 +254,14 @@ static int s1_init(void)
}
OBJ_DESTRUCT(&kv);
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
s1_pname.jobid = strtoul(pmix_id, &str, 10);
s1_pname.jobid = (s1_pname.jobid << 16) & 0xffff0000;
if (NULL != str) {
ui32 = strtoul(str, NULL, 10);
s1_pname.jobid |= (ui32 & 0x0000ffff);
}
free(pmix_id);
ldr.jobid = s1_pname.jobid;
s1_pname.vpid = rank;
opal_proc_set_name(&s1_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s1: assigned tmp name",
OPAL_NAME_PRINT(s1_pname));
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
if (pmix_kvs_name == NULL) {
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI_KVS_Get_my_name(pmix_kvs_name, pmix_kvslen_max);
if( PMI_SUCCESS != rc ) {
if (PMI_SUCCESS != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get_my_name");
goto err_exit;
}
@ -606,17 +604,14 @@ static int s1_get(const opal_process_name_t *id,
return rc;
}
static int s1_publish(opal_pmix_data_range_t scope,
opal_pmix_persistence_t persist,
opal_list_t *info)
static int s1_publish(opal_list_t *info)
{
// SLURM PMIv1 doesn't implement this function
return OPAL_ERR_NOT_SUPPORTED;
}
static int s1_lookup(opal_pmix_data_range_t scope,
opal_list_t *data)
static int s1_lookup(opal_list_t *data, opal_list_t *info)
{
// Allocate mem for port here? Otherwise we won't get success!
// SLURM PMIv1 doesn't implement this function
@ -624,7 +619,7 @@ static int s1_lookup(opal_pmix_data_range_t scope,
return OPAL_ERR_NOT_SUPPORTED;
}
static int s1_unpublish(opal_pmix_data_range_t scope, char **keys)
static int s1_unpublish(char **keys, opal_list_t *info)
{
// SLURM PMIv1 doesn't implement this function

156
opal/mca/pmix/s2/pmix_s2.c
Просмотреть файл

@ -48,12 +48,9 @@ static int s2_put(opal_pmix_scope_t scope,
static int s2_get(const opal_process_name_t *id,
const char *key,
opal_value_t **kv);
static int s2_publish(opal_pmix_data_range_t scope,
opal_pmix_persistence_t persist,
opal_list_t *info);
static int s2_lookup(opal_pmix_data_range_t scope,
opal_list_t *data);
static int s2_unpublish(opal_pmix_data_range_t scope, char **keys);
static int s2_publish(opal_list_t *info);
static int s2_lookup(opal_list_t *data, opal_list_t *info);
static int s2_unpublish(char **keys, opal_list_t *info);
static int s2_spawn(opal_list_t *jobinfo, opal_list_t *apps, opal_jobid_t *jobid);
static int s2_job_connect(opal_list_t *procs);
static int s2_job_disconnect(opal_list_t *procs);
@ -154,8 +151,15 @@ static int kvs_get(const char key[], char value [], int maxvalue)
int rc;
int len;
rc = PMI2_KVS_Get(pmix_kvs_name, PMI2_ID_NULL, key, value, maxvalue, &len);
if( PMI2_SUCCESS != rc || len < 0){
OPAL_PMI_ERROR(rc, "PMI2_KVS_Get");
/*
* turns out the KVS can be called for keys that haven't yet
* been inserted, so suppress warning message if this is the
* case
*/
if (PMI_SUCCESS != rc) {
if (PMI_ERR_INVALID_KEY != rc) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Get");
}
return OPAL_ERROR;
}
return OPAL_SUCCESS;
@ -183,6 +187,9 @@ static int s2_init(void)
size = -1;
rank = -1;
appnum = -1;
// setup hash table so we always can finalize it
opal_pmix_base_hash_init();
if (PMI2_SUCCESS != (rc = PMI2_Init(&spawned, &size, &rank, &appnum))) {
opal_show_help("help-pmix-base.txt", "pmix2-init-failed", true, rc);
return OPAL_ERROR;
@ -196,8 +203,56 @@ static int s2_init(void)
s2_rank = rank;
s2_appnum = appnum;
// setup hash table
opal_pmix_base_hash_init();
pmix_vallen_max = PMI2_MAX_VALLEN;
pmix_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmix_keylen_max = PMI2_MAX_KEYLEN;
pmix_vallen_threshold = PMI2_MAX_VALLEN * 3;
pmix_vallen_threshold >>= 2;
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmix_kvs_name, pmix_kvslen_max);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
free(pmix_kvs_name);
goto err_exit;
}
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
s2_pname.jobid = strtoul(pmix_kvs_name, &str, 10);
s2_pname.jobid = (s2_pname.jobid << 16) & 0xffff0000;
if (NULL != str) {
stepid = strtoul(str, NULL, 10);
s2_pname.jobid |= (stepid & 0x0000ffff);
}
s2_pname.vpid = s2_rank;
opal_proc_set_name(&s2_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s2: assigned tmp name",
OPAL_NAME_PRINT(s2_pname));
/* Slurm PMI provides the job id as an integer followed
* by a '.', followed by essentially a stepid. The first integer
* defines an overall job number. The second integer is the number of
* individual jobs we have run within that allocation.
*/
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_STRING;
kv.data.string = pmix_kvs_name;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv); // frees pmix_kvs_name
/* save the job size */
OBJ_CONSTRUCT(&kv, opal_value_t);
@ -223,12 +278,6 @@ static int s2_init(void)
}
OBJ_DESTRUCT(&kv);
pmix_vallen_max = PMI2_MAX_VALLEN;
pmix_kvslen_max = PMI2_MAX_VALLEN; // FIX ME: What to put here for versatility?
pmix_keylen_max = PMI2_MAX_KEYLEN;
pmix_vallen_threshold = PMI2_MAX_VALLEN * 3;
pmix_vallen_threshold >>= 2;
rc = PMI2_Info_GetJobAttr("universeSize", buf, 16, &found);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI_Get_universe_size");
@ -246,50 +295,6 @@ static int s2_init(void)
}
OBJ_DESTRUCT(&kv);
pmix_kvs_name = (char*)malloc(pmix_kvslen_max);
if( pmix_kvs_name == NULL ){
PMI2_Finalize();
ret = OPAL_ERR_OUT_OF_RESOURCE;
goto err_exit;
}
rc = PMI2_Job_GetId(pmix_kvs_name, pmix_kvslen_max);
if( PMI2_SUCCESS != rc ) {
OPAL_PMI_ERROR(rc, "PMI2_Job_GetId");
goto err_exit;
}
/* Slurm PMI provides the job id as an integer followed
* by a '.', followed by essentially a stepid. The first integer
* defines an overall job number. The second integer is the number of
* individual jobs we have run within that allocation.
*/
OBJ_CONSTRUCT(&kv, opal_value_t);
kv.key = strdup(OPAL_PMIX_JOBID);
kv.type = OPAL_STRING;
kv.data.string = pmix_kvs_name;
if (OPAL_SUCCESS != (ret = opal_pmix_base_store(&OPAL_PROC_MY_NAME, &kv))) {
OPAL_ERROR_LOG(ret);
OBJ_DESTRUCT(&kv);
goto err_exit;
}
OBJ_DESTRUCT(&kv);
/* store our name in the opal_proc_t so that
* debug messages will make sense - an upper
* layer will eventually overwrite it, but that
* won't do any harm */
s2_pname.jobid = strtoul(pmix_kvs_name, &str, 10);
s2_pname.jobid = (s2_pname.jobid << 16) & 0xffff0000;
if (NULL != str) {
stepid = strtoul(str, NULL, 10);
s2_pname.jobid |= (stepid & 0x0000ffff);
}
s2_pname.vpid = s2_rank;
opal_proc_set_name(&s2_pname);
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s pmix:s2: assigned tmp name",
OPAL_NAME_PRINT(s2_pname));
char *pmapping = (char*)malloc(PMI2_MAX_VALLEN);
if( pmapping == NULL ){
rc = OPAL_ERR_OUT_OF_RESOURCE;
@ -531,17 +536,6 @@ static int s2_put(opal_pmix_scope_t scope,
static int s2_commit(void)
{
int rc;
/* check if there is partially filled meta key and put them */
opal_pmix_base_commit_packed (&pmix_packed_data, &pmix_packed_data_offset,
&pmix_packed_encoded_data, &pmix_packed_encoded_data_offset,
pmix_vallen_max, &pmix_pack_key, kvs_put);
if (PMI_SUCCESS != (rc = PMI_KVS_Commit(pmix_kvs_name))) {
OPAL_PMI_ERROR(rc, "PMI_KVS_Commit");
return OPAL_ERROR;
}
return OPAL_SUCCESS;
}
@ -557,6 +551,16 @@ static int s2_fence(opal_list_t *procs, int collect_data)
"%s pmix:s2 called fence",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* check if there is partially filled meta key and put them */
opal_pmix_base_commit_packed (&pmix_packed_data, &pmix_packed_data_offset,
&pmix_packed_encoded_data, &pmix_packed_encoded_data_offset,
pmix_vallen_max, &pmix_pack_key, kvs_put);
/* now call fence */
if (PMI2_SUCCESS != PMI2_KVS_Fence()) {
return OPAL_ERROR;
}
/* get the modex data from each local process and set the
* localities to avoid having the MPI layer fetch data
* for every process in the job */
@ -610,13 +614,12 @@ static int s2_get(const opal_process_name_t *id,
opal_value_t **kv)
{
int rc;
opal_output(0, "CALLED GET FOR %s", key);
rc = opal_pmix_base_cache_keys_locally(id, key, kv, pmix_kvs_name, pmix_vallen_max, kvs_get);
return rc;
}
static int s2_publish(opal_pmix_data_range_t scope,
opal_pmix_persistence_t persist,
opal_list_t *info)
static int s2_publish(opal_list_t *info)
{
#if 0
int rc;
@ -629,8 +632,7 @@ static int s2_publish(opal_pmix_data_range_t scope,
return OPAL_ERR_NOT_IMPLEMENTED;
}
static int s2_lookup(opal_pmix_data_range_t scope,
opal_list_t *data)
static int s2_lookup(opal_list_t *data, opal_list_t *info)
{
#if 0
int rc;
@ -644,7 +646,7 @@ static int s2_lookup(opal_pmix_data_range_t scope,
return OPAL_ERR_NOT_IMPLEMENTED;
}
static int s2_unpublish(opal_pmix_data_range_t scope, char **keys)
static int s2_unpublish(char **keys, opal_list_t *info)
{
#if 0
int rc;

3
orte/mca/ess/base/ess_base_std_orted.c
Просмотреть файл

@ -510,7 +510,8 @@ int orte_ess_base_orted_setup(char **hosts)
/* obviously, we have "reported" */
jdata->num_reported = 1;
/* setup the PMIx framework */
/* setup the PMIx framework - ensure it skips all non-PMIx components */
putenv("OMPI_MCA_pmix=^s1,s2,cray");
if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_pmix_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_pmix_base_open";

3
orte/mca/ess/hnp/ess_hnp_module.c
Просмотреть файл

@ -628,7 +628,8 @@ static int rte_init(void)
free(contact_path);
}
/* setup the PMIx framework */
/* setup the PMIx framework - ensure it skips all non-PMIx components */
putenv("OMPI_MCA_pmix=^s1,s2,cray");
if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_pmix_base_framework, 0))) {
ORTE_ERROR_LOG(ret);
error = "orte_pmix_base_open";

3
orte/mca/schizo/ompi/schizo_ompi.c
Просмотреть файл

@ -425,6 +425,9 @@ static int setup_fork(orte_job_t *jdata,
*/
opal_setenv("OMPI_MCA_ess", "pmi", false, &app->env);
/* ensure that the spawned process ignores direct launch components */
opal_setenv("OMPI_MCA_pmix", "^s1,s2,cray", true, &app->env);
/* since we want to pass the name as separate components, make sure
* that the "name" environmental variable is cleared!
*/