oshmem/proc: Refactor oshmem_proc to meet new add_proc changes

ompi has new mpi_add_procs_cutoff argument that can control
creation of ompi_proc_t but We should be confident that all
ompi_proc_t object exists during oshmem_group_all creation.
Probably it could be done in more flexible way later.

Signed-off-by: Igor Ivanov <Igor.Ivanov@itseez.com>

oshmem/proc/proc.c

@@ -60,6 +60,13 @@ OBJ_CLASS_INSTANCE(oshmem_group_t, opal_object_t, NULL, NULL);
 
 int oshmem_proc_group_init(void)
 {
+    if (orte_process_info.num_procs != opal_list_get_size(&ompi_proc_list)) {
+        opal_output(0,
+                "Error: oshmem_group_all is not created: orte_process_info.num_procs = %d ompi_proc_list = %d",
+		orte_process_info.num_procs,
+		opal_list_get_size(&ompi_proc_list));
+        return OSHMEM_ERROR;
+    }
 
     /* Setup communicator array */
     OBJ_CONSTRUCT(&oshmem_group_array, opal_pointer_array_t);
@@ -76,7 +83,7 @@ int oshmem_proc_group_init(void)
             == (oshmem_group_all =
                     oshmem_proc_group_create(0,
                                              1,
-                                             opal_list_get_size(&ompi_proc_list)))) {
+					     oshmem_num_procs()))) {
         oshmem_proc_group_destroy(oshmem_group_all);
         return OSHMEM_ERROR;
     }
@@ -128,6 +135,8 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start,
     oshmem_proc_t** proc_array = NULL;
     oshmem_proc_t* proc = NULL;
 
+    assert(oshmem_proc_local());
+
     group = OBJ_NEW(oshmem_group_t);
 
     if (group) {
@@ -145,10 +154,13 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start,
 
         group->my_pe = oshmem_proc_pe(oshmem_proc_local());
         group->is_member = 0;
-        /* now save only the procs that match this jobid */
-        for (proc = (oshmem_proc_t*) opal_list_get_first(&ompi_proc_list);
-                proc != (oshmem_proc_t*) opal_list_get_end(&ompi_proc_list);
-                proc = (oshmem_proc_t*) opal_list_get_next(proc)) {
+        for (i = 0 ; i < oshmem_num_procs() ; i++) {
+            proc = oshmem_proc_find(i);
+            if (NULL == proc) {
+                opal_output(0,
+                             "Error: Can not find proc object for pe = %d", i);
+                return NULL;
+            }
             if (count_pe >= (int) pe_size) {
                 break;
             } else if ((cur_pe >= pe_start)

oshmem/proc/proc.h

@@ -129,7 +129,7 @@ OSHMEM_DECLSPEC int oshmem_proc_finalize(void);
  *
  * @return Pointer to the local process structure
  */
-static inline oshmem_proc_t* oshmem_proc_local(void)
+static inline oshmem_proc_t *oshmem_proc_local(void)
 {
     return (oshmem_proc_t *)ompi_proc_local_proc;
 }
@@ -145,9 +145,18 @@ static inline oshmem_proc_t* oshmem_proc_local(void)
  *
  * @return Pointer to the process instance for \c name
  */
-static inline oshmem_proc_t * oshmem_proc_find(const orte_process_name_t* name)
+static inline oshmem_proc_t *oshmem_proc_for_find(const orte_process_name_t name)
 {
-	return (oshmem_proc_t *)ompi_proc_find(name);
+    return (oshmem_proc_t *)ompi_proc_for_name((const opal_process_name_t)name);
+}
+
+static inline oshmem_proc_t *oshmem_proc_find(int pe)
+{
+    orte_process_name_t name;
+
+    name.jobid = ORTE_PROC_MY_NAME->jobid;
+    name.vpid = pe;
+    return oshmem_proc_for_find(name);
 }
 
 static inline int oshmem_proc_pe(oshmem_proc_t *proc)
@@ -203,7 +212,7 @@ OSHMEM_DECLSPEC int oshmem_proc_group_finalize(void);
  * @return Array of pointers to proc instances in the current
  * known universe, or NULL if there is an internal failure.
  */
-OSHMEM_DECLSPEC oshmem_group_t* oshmem_proc_group_create(int pe_start,
+OSHMEM_DECLSPEC oshmem_group_t *oshmem_proc_group_create(int pe_start,
                                                          int pe_stride,
                                                          size_t pe_size);
 
@@ -218,7 +227,7 @@ static inline oshmem_proc_t *oshmem_proc_group_all(int pe)
     return oshmem_group_all->proc_array[pe];
 }
 
-static inline oshmem_proc_t* oshmem_proc_group_find(oshmem_group_t* group,
+static inline oshmem_proc_t *oshmem_proc_group_find(oshmem_group_t* group,
                                                     int pe)
 {
     int i = 0;
@@ -241,7 +250,7 @@ static inline oshmem_proc_t* oshmem_proc_group_find(oshmem_group_t* group,
 
         name.jobid = ORTE_PROC_MY_NAME->jobid;
         name.vpid = pe;
-        proc = oshmem_proc_find(&name);
+        proc = oshmem_proc_for_find(name);
     }
 
     return proc;
@@ -271,10 +280,8 @@ static inline int oshmem_proc_group_is_member(oshmem_group_t *group)
 
 static inline int oshmem_num_procs(void)
 {
-    if (!oshmem_group_all)
-        return opal_list_get_size(&ompi_proc_list);
-
-    return oshmem_group_all->proc_count;
+    return (oshmem_group_all ?
+        oshmem_group_all->proc_count : opal_list_get_size(&ompi_proc_list));
 }
 
 static inline int oshmem_my_proc_id(void)