From 6ba25951b4be4c47dfc2273d357f8325aa5b0a36 Mon Sep 17 00:00:00 2001 From: Shiqing Fan Date: Fri, 26 Jun 2009 13:35:45 +0000 Subject: [PATCH] Add a separate function for reading remote registry, so that it could be easily reused. Add two options for plm process module, i.e. remote_env_prefix for getting OMPI prefix on remote computer by reading its user environment variable (OPENMPI_HOME), and remote_reg_prefix is similar, but it reads the registry on the remote computer. Reading remote env prefix has a higher priority than reading reg prefix, so that user can use their own installation of OMPI. This commit was SVN r21538. --- orte/mca/plm/ccp/plm_ccp_module.c | 2 +- orte/mca/plm/process/plm_process.h | 4 +- orte/mca/plm/process/plm_process_component.c | 10 +- orte/mca/plm/process/plm_process_module.c | 376 +++++++++++-------- 4 files changed, 224 insertions(+), 168 deletions(-) diff --git a/orte/mca/plm/ccp/plm_ccp_module.c b/orte/mca/plm/ccp/plm_ccp_module.c index 0f6fc499f6..635050f045 100644 --- a/orte/mca/plm/ccp/plm_ccp_module.c +++ b/orte/mca/plm/ccp/plm_ccp_module.c @@ -698,7 +698,7 @@ static int plm_ccp_disconnect(void) static char *plm_ccp_commandline(char *prefix, char *node_name, int argc, char **argv) { char *commandline; - size_t i, len = 0; + int i, len = 0; for( i = 0; i < argc; i++ ) { len += strlen(argv[i]) + 1; diff --git a/orte/mca/plm/process/plm_process.h b/orte/mca/plm/process/plm_process.h index b14cd2003f..df09297ae3 100644 --- a/orte/mca/plm/process/plm_process.h +++ b/orte/mca/plm/process/plm_process.h @@ -5,7 +5,7 @@ * Copyright (c) 2004-2006 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2009 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. @@ -59,6 +59,8 @@ struct orte_plm_process_component_t { bool assume_same_shell; bool force_process; bool use_gui_prompt; + bool remote_reg_prefix; + bool remote_env_prefix; int delay; int priority; char* orted; diff --git a/orte/mca/plm/process/plm_process_component.c b/orte/mca/plm/process/plm_process_component.c index 0bbe3806fb..a479c2ce37 100644 --- a/orte/mca/plm/process/plm_process_component.c +++ b/orte/mca/plm/process/plm_process_component.c @@ -147,8 +147,16 @@ int orte_plm_process_component_open(void) mca_plm_process_component.assume_same_shell = OPAL_INT_TO_BOOL(tmp); mca_base_param_reg_int(c, "use_gui_prompt", "If set to 1, use Windows standard GUI to input user name and password for connecting to remote node. Otherwise, use command line prompt.", - false, false, 1, &tmp); + false, false, 0, &tmp); mca_plm_process_component.use_gui_prompt = OPAL_INT_TO_BOOL(tmp); + mca_base_param_reg_int(c, "remote_reg_prefix", + "If set to 1, the process module will first try to read OPAL_PREFIX registry entry on remote computer to get the orte daemon execute path. If the read failed, it will use the default prefix.", + false, false, 1, &tmp); + mca_plm_process_component.remote_reg_prefix = OPAL_INT_TO_BOOL(tmp); + mca_base_param_reg_int(c, "remote_env_prefix", + "If set to 1, the process module will first try to read OPENMPI_HOME user env on remote computer to get the orte daemon execute path. If the read failed, it will try to read remote registry, and then default prefix.", + false, false, 1, &tmp); + mca_plm_process_component.remote_env_prefix = OPAL_INT_TO_BOOL(tmp); return ORTE_SUCCESS; } diff --git a/orte/mca/plm/process/plm_process_module.c b/orte/mca/plm/process/plm_process_module.c index a23e089649..ecf0c092c7 100644 --- a/orte/mca/plm/process/plm_process_module.c +++ b/orte/mca/plm/process/plm_process_module.c @@ -63,9 +63,6 @@ #include #include #include -#include -#include -#include #pragma comment(lib, "wbemuuid.lib") #pragma comment(lib, "comsuppw.lib") @@ -156,9 +153,10 @@ static const char * orte_plm_process_shell_name[] = { /* * local functions */ -static char *generate_commandline(char *, int, char **); -static int wmi_launch_child(char *, char *, int, char **); -static int get_credential(char *, char *, char *); +static char *generate_commandline(char *prefix, int argc, char **argv); +static int wmi_launch_child(char *prefix, char *remote_node, int argc, char **argv); +static int get_credential(char *node_name); +static char *read_remote_registry(uint32_t root, char *sub_key, char *key, char *remote_node, char *ntlm_auth); /* local global storage of timing variables */ @@ -168,6 +166,15 @@ static struct timeval joblaunchstart, joblaunchstop; static char user_name[CREDUI_MAX_USERNAME_LENGTH+1]; static char user_password[CREDUI_MAX_PASSWORD_LENGTH+1]; +/* WMI service locator */ +IWbemLocator *pLoc = NULL; +/* namespace for \hostname\root\default */ +IWbemServices *pSvc_registry = NULL; +/* namespace for \hostname\root\cimv2 */ +IWbemServices *pSvc_cimv2 = NULL; +/* Security levels on a WMI connection */ +SEC_WINNT_AUTH_IDENTITY cID; + /* global storage of active jobid being launched */ static orte_jobid_t active_job = ORTE_JOBID_INVALID; @@ -207,6 +214,18 @@ int orte_plm_process_init(void) CoUninitialize(); return ORTE_ERROR; } + + /* Obtain the initial locator to WMI. */ + hres = CoCreateInstance(CLSID_WbemLocator, + 0, + CLSCTX_INPROC_SERVER, + IID_IWbemLocator, (LPVOID *) &pLoc); + + if (FAILED(hres)) { + opal_output(0,"Failed to create IWbemLocator object. Err code = %d \n", hres); + CoUninitialize(); + return ORTE_ERROR; + } SecureZeroMemory(user_name, sizeof(user_name)); SecureZeroMemory(user_password, sizeof(user_password)); @@ -230,12 +249,12 @@ static char *generate_commandline(char *prefix, int argc, char **argv) } } - commandline = (char*)malloc( len + strlen(prefix) + 3); + commandline = (char*)malloc( len + strlen(prefix) + 13); memset(commandline, '\0', strlen(commandline)); strcat(commandline, "\""); strcat(commandline, prefix); - strcat(commandline, "\" "); + strcat(commandline, "\\bin\\orted\" "); for(i=1;iConnectServer(_com_util::ConvertStringToBSTR(namespace_default), /* namespace */ - _com_util::ConvertStringToBSTR(user_name), /* User name */ - _com_util::ConvertStringToBSTR(user_password), /* User password */ - (L"MS_409"), /* Locale */ - NULL, /* Security flags */ - (L"ntlmdomain:domain"), /* Authority */ - 0, /* Context object */ - &pSvc_registry /* IWbemServices proxy */ - ); + if( NULL == pSvc_registry) { + /* connect to default namespace */ + hres = pLoc->ConnectServer(_com_util::ConvertStringToBSTR(namespace_default), /* namespace */ + _com_util::ConvertStringToBSTR(user_name), /* User name */ + _com_util::ConvertStringToBSTR(user_password), /* User password */ + (L"MS_409"), /* Locale */ + NULL, /* Security flags */ + _com_util::ConvertStringToBSTR(ntlm_auth), /* Authority */ + 0, /* Context object */ + &pSvc_registry /* IWbemServices proxy */ + ); - if (FAILED(hres)) { - opal_output(0,"Could not connect to namespace DEFAULT on node %s. Error code = %d \n", - remote_node, hres); - pLoc->Release(); - CoUninitialize(); - return ORTE_ERROR; - } + if (FAILED(hres)) { + opal_output(0,"Could not connect to namespace DEFAULT on node %s. Error code = %d \n", + remote_node, hres); + return NULL; + } - OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output, - "%s plm:process: Connected to \\\\%s\\\\ROOT\\\\DEFAULT", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), - remote_node)); - - /* connect to cinv2 namespace */ - hres = pLoc->ConnectServer(_com_util::ConvertStringToBSTR(namespace_cimv2), /* namespace */ - _com_util::ConvertStringToBSTR(user_name), /* User name */ - _com_util::ConvertStringToBSTR(user_password), /* User password */ - (L"MS_409"), /* Locale */ - NULL, /* Security flags */ - (L"ntlmdomain:domain"), /* Authority */ - 0, /* Context object */ - &pSvc_cimv2 /* IWbemServices proxy */ - ); + OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output, + "%s plm:process: Connected to \\\\%s\\\\ROOT\\\\DEFAULT", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), + remote_node)); - if (FAILED(hres)) { - opal_output(0,"Could not connect to namespace cimv2 on node %s. Error code =%d \n", - remote_node, hres); - pLoc->Release(); - pSvc_registry->Release(); - CoUninitialize(); - return ORTE_ERROR; + /* default namespace */ + hres = CoSetProxyBlanket(pSvc_registry, /* Indicates the proxy to set */ + RPC_C_AUTHN_WINNT, /* RPC_C_AUTHN_xxx */ + RPC_C_AUTHZ_NONE, /* RPC_C_AUTHZ_xxx */ + NULL, /* Server principal name */ + RPC_C_AUTHN_LEVEL_CALL, /* RPC_C_AUTHN_LEVEL_xxx */ + RPC_C_IMP_LEVEL_IMPERSONATE, /* RPC_C_IMP_LEVEL_xxx */ + &cID, /* client identity */ + EOAC_NONE /* proxy capabilities */ + ); + + if (FAILED(hres)) { + opal_output(0,"Could not set proxy blanket. Error code = %d \n", hres); + return NULL; + } } BSTR ClassName_registry = SysAllocString(L"StdRegProv"); BSTR MethodName_registry = SysAllocString(L"GetStringValue"); - BSTR MethodName_cimv2 = SysAllocString(L"Create"); - BSTR ClassName_cimv2 = SysAllocString(L"Win32_Process"); - OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output, - "%s plm:process: Connected to \\\\%s\\\\ROOT\\\\CIMV2", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), - remote_node)); - - /* Set security levels on a WMI connection */ - SEC_WINNT_AUTH_IDENTITY cID; - cID.User = (unsigned char *) user_name; - cID.UserLength = lstrlen(user_name); - cID.Password = (unsigned char *) user_password; - cID.PasswordLength = lstrlen(user_password); - cID.Domain = (unsigned char *) remote_node; - cID.DomainLength = lstrlen(remote_node); - cID.Flags = SEC_WINNT_AUTH_IDENTITY_ANSI; - - /* default namespace */ - hres = CoSetProxyBlanket(pSvc_registry, /* Indicates the proxy to set */ - RPC_C_AUTHN_WINNT, /* RPC_C_AUTHN_xxx */ - RPC_C_AUTHZ_NONE, /* RPC_C_AUTHZ_xxx */ - NULL, /* Server principal name */ - RPC_C_AUTHN_LEVEL_CALL, /* RPC_C_AUTHN_LEVEL_xxx */ - RPC_C_IMP_LEVEL_IMPERSONATE, /* RPC_C_IMP_LEVEL_xxx */ - &cID, /* client identity */ - EOAC_NONE /* proxy capabilities */ - ); - - if (FAILED(hres)) { - opal_output(0,"Could not set proxy blanket. Error code = %d \n", hres); - goto cleanup; - } - - /* cimv2 namespace */ - hres = CoSetProxyBlanket(pSvc_cimv2, /* Indicates the proxy to set */ - RPC_C_AUTHN_WINNT, /* RPC_C_AUTHN_xxx */ - RPC_C_AUTHZ_NONE, /* RPC_C_AUTHZ_xxx */ - NULL, /* Server principal name */ - RPC_C_AUTHN_LEVEL_CALL, /* RPC_C_AUTHN_LEVEL_xxx */ - RPC_C_IMP_LEVEL_IMPERSONATE, /* RPC_C_IMP_LEVEL_xxx */ - &cID, /* client identity */ - EOAC_NONE /* proxy capabilities */ - ); - - if (FAILED(hres)) { - opal_output(0,"Could not set proxy blanket. Error code = %d \n", hres ); - goto cleanup; - } - IWbemClassObject* pClass_registry = NULL; hres = pSvc_registry->GetObject(ClassName_registry, 0, NULL, &pClass_registry, NULL); if (FAILED(hres)) { opal_output(0,"Could not get Wbem class object. Error code = %d \n", hres); - goto cleanup; + return NULL; } IWbemClassObject* pInParamsDefinition_registry = NULL; @@ -517,32 +444,37 @@ static int wmi_launch_child(char *prefix, char *remote_node, int argc, char **ar IWbemClassObject* pClassInstance_registry = NULL; hres = pInParamsDefinition_registry->SpawnInstance(0, &pClassInstance_registry); - /* set registry path, HKEY_LOCAL_MACHINE is default */ + + VARIANT hkey_root; + hkey_root.vt = VT_I4; + hkey_root.intVal = root; + + hres = pClassInstance_registry->Put(L"hDefKey", 0, &hkey_root, 0); + + VARIANT varSubKeyName; varSubKeyName.vt = VT_BSTR; - varSubKeyName.bstrVal = L"Software\\Open MPI\\"; + varSubKeyName.bstrVal = _com_util::ConvertStringToBSTR(sub_key); - hres = pClassInstance_registry->Put(L"sSubKeyName", 0, - &varSubKeyName, 0); + hres = pClassInstance_registry->Put(L"sSubKeyName", 0, &varSubKeyName, 0); if(FAILED(hres)) { opal_output(0,"Could not Store the value for the in parameters. Error code = %d \n",hres); pClass_registry->Release(); pInParamsDefinition_registry->Release(); - goto cleanup; + return NULL; } VARIANT varsValueName; varsValueName.vt = VT_BSTR; - varsValueName.bstrVal = L"OPAL_PREFIX"; + varsValueName.bstrVal = _com_util::ConvertStringToBSTR(key); - hres = pClassInstance_registry->Put(L"sValueName", 0, - &varsValueName, 0); + hres = pClassInstance_registry->Put(L"sValueName", 0, &varsValueName, 0); if(FAILED(hres)) { opal_output(0,"Could not Store the value for the in parameters. Error code = %d \n",hres); pClass_registry->Release(); pInParamsDefinition_registry->Release(); - goto cleanup; + return NULL; } /* Execute Method to read OPAL_PREFIX in the registry */ @@ -557,7 +489,7 @@ static int wmi_launch_child(char *prefix, char *remote_node, int argc, char **ar if (FAILED(hres)) { pOutParams_registry->Release(); opal_output(0,"Could not execute method. Error code = %d \n",hres); - goto cleanup; + return NULL; } /* To see what the method returned */ @@ -571,13 +503,126 @@ static int wmi_launch_child(char *prefix, char *remote_node, int argc, char **ar &varsValue_registry, NULL, 0); pOutParams_registry->Release(); - if( VT_NULL == varsValue_registry.vt) { - command_line = generate_commandline(prefix, argc, argv); + if( VT_NULL != varsValue_registry.vt) { + char *value = strdup(_com_util::ConvertBSTRToString(varsValue_registry.bstrVal)); + return value; } else { - char *reg_prefix = (char *) malloc(sizeof(char)*(strlen(_com_util::ConvertBSTRToString(varsValue_registry.bstrVal))+10)); - strcpy(reg_prefix, _com_util::ConvertBSTRToString(varsValue_registry.bstrVal)); - strcat(reg_prefix, "\\bin\\orted"); - command_line = generate_commandline(reg_prefix, argc, argv); + return NULL; + } + +} + + +/** + * Remote spawn process using WMI. + */ +static int wmi_launch_child(char *prefix, char *remote_node, int argc, char **argv) +{ + char *command_line = NULL; + int len = 0, pid = -1; + + HRESULT hres; + + /*Connect to WMI through the IWbemLocator::ConnectServer method*/ + char namespace_cimv2[100]; + + char *ntlm_auth = (char *) malloc(sizeof(char)*(strlen("ntlmdomain:")+strlen(remote_node)+1)); + memset(ntlm_auth, 0, strlen(ntlm_auth)); + strcat(ntlm_auth, "ntlmdomain:"); + strcat(ntlm_auth, remote_node); + + if( 0 != get_credential(remote_node)) { + return ORTE_ERROR; + } + + /* set up remote namespace path */ + char *rt_namespace_cimv2 = "\\root\\cimv2"; + strcpy(namespace_cimv2, "\\\\"); + strcat(namespace_cimv2, remote_node ); + strcat(namespace_cimv2, rt_namespace_cimv2); + + /* connect to cimv2 namespace */ + hres = pLoc->ConnectServer(_com_util::ConvertStringToBSTR(namespace_cimv2), /* namespace */ + _com_util::ConvertStringToBSTR(user_name), /* User name */ + _com_util::ConvertStringToBSTR(user_password), /* User password */ + (L"MS_409"), /* Locale */ + NULL, /* Security flags */ + _com_util::ConvertStringToBSTR(ntlm_auth), /* Authority */ + 0, /* Context object */ + &pSvc_cimv2 /* IWbemServices proxy */ + ); + + if (FAILED(hres)) { + opal_output(0,"Could not connect to namespace cimv2 on node %s. Error code =%d \n", + remote_node, hres); + return ORTE_ERROR; + } + + OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output, + "%s plm:process: Connected to \\\\%s\\\\ROOT\\\\CIMV2", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), + remote_node)); + + BSTR MethodName_cimv2 = SysAllocString(L"Create"); + BSTR ClassName_cimv2 = SysAllocString(L"Win32_Process"); + + /* Set security levels on a WMI connection */ + cID.User = (unsigned char *) user_name; + cID.UserLength = lstrlen(user_name); + cID.Password = (unsigned char *) user_password; + cID.PasswordLength = lstrlen(user_password); + cID.Domain = (unsigned char *) remote_node; + cID.DomainLength = lstrlen(remote_node); + cID.Flags = SEC_WINNT_AUTH_IDENTITY_ANSI; + + /* cimv2 namespace */ + hres = CoSetProxyBlanket(pSvc_cimv2, /* Indicates the proxy to set */ + RPC_C_AUTHN_WINNT, /* RPC_C_AUTHN_xxx */ + RPC_C_AUTHZ_NONE, /* RPC_C_AUTHZ_xxx */ + NULL, /* Server principal name */ + RPC_C_AUTHN_LEVEL_CALL, /* RPC_C_AUTHN_LEVEL_xxx */ + RPC_C_IMP_LEVEL_IMPERSONATE, /* RPC_C_IMP_LEVEL_xxx */ + &cID, /* client identity */ + EOAC_NONE /* proxy capabilities */ + ); + + if (FAILED(hres)) { + opal_output(0,"Could not set proxy blanket. Error code = %d \n", hres ); + return pid; + } + + /* if there isn't a prefix ( e.g., '--noprefix' specfied, + or Open MPI was configured without ORTE_WANT_ORTERUN_PREFIX_BY_DEFAULT), + let's first check the OPENMPI_HOME in user environment variables, + and then the OPAL_PREFIX registry value. */ + if( NULL == prefix ) { + if( mca_plm_process_component.remote_env_prefix ) { + char *path = "Environment"; + char *key = "OPENMPI_HOME"; + /* read registry at HKEY_CURRENT_USER + please note: this MUST be the same user as for WMI authorization. */ + char *reg_prefix = read_remote_registry(0x80000001, path, key, remote_node, ntlm_auth); + if( NULL != reg_prefix) { + command_line = generate_commandline(reg_prefix, argc, argv); + } + } + if( NULL == command_line && mca_plm_process_component.remote_reg_prefix ) { + char *path = "Software\\Open MPI\\"; + char *key = "OPAL_PREFIX"; + char *reg_prefix = read_remote_registry(0x80000002, path, key, remote_node, ntlm_auth); + if( NULL != reg_prefix) { + command_line = generate_commandline(reg_prefix, argc, argv); + } + } + } else { + /* use user specified/default prefix */ + command_line = generate_commandline(prefix, argc, argv); + } + + if ( NULL == command_line ) { + /* we couldn't find the execute path, abort. */ + opal_output(0, "couldn't find executable path for orted on node %s. aborted.", remote_node); + return ORTE_ERR_NOT_FOUND; } /* Use the IWbemServices pointer to make requests of WMI */ @@ -634,14 +679,8 @@ static int wmi_launch_child(char *prefix, char *remote_node, int argc, char **ar pOutParams_cimv2->Release(); cleanup: - pLoc->Release(); - pSvc_registry->Release(); - pSvc_cimv2->Release(); - SysFreeString(ClassName_registry); - SysFreeString(MethodName_registry); SysFreeString(ClassName_cimv2); SysFreeString(MethodName_cimv2); - CoUninitialize(); return pid; } @@ -1268,7 +1307,7 @@ int orte_plm_process_launch(orte_job_t *jdata) } /* launch remote process */ - pid = wmi_launch_child(exec_path, nodes[nnode]->name, argc, exec_argv); + pid = wmi_launch_child(prefix_dir, nodes[nnode]->name, argc, exec_argv); if (pid < 0) { failed_launch = true; @@ -1393,6 +1432,13 @@ int orte_plm_process_signal_job(orte_jobid_t jobid, int32_t signal) int orte_plm_process_finalize(void) { int rc; + + /* release the locator and service objects*/ + pLoc->Release(); + pSvc_registry->Release(); + pSvc_cimv2->Release(); + + CoUninitialize(); /* cleanup any pending recvs */ if (ORTE_SUCCESS != (rc = orte_plm_base_comm_stop())) {