pmix/ext11: correctly use PMIx_server_register_nspace()

PMIx_server_register_nspace() is an asynchronous operation, so
the pmix glue wait for it completes before returning.

Signed-off-by: Gilles Gouaillardet <gilles@rist.or.jp>

opal/mca/pmix/ext11/pmix_ext.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
+ * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
  *                         All rights reserved.
@@ -527,6 +527,7 @@ static void opcon(pmix1_opcaddy_t *p)
     p->ninfo = 0;
     p->apps = NULL;
     p->sz = 0;
+    p->active = false;
     p->opcbfunc = NULL;
     p->mdxcbfunc = NULL;
     p->valcbfunc = NULL;

opal/mca/pmix/ext11/pmix_ext.h

@@ -2,7 +2,7 @@
  * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2015 Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016      Research Organization for Information Science
+ * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -62,6 +62,7 @@ typedef struct {
     size_t ninfo;
     pmix_app_t *apps;
     size_t sz;
+    volatile bool active;
     opal_pmix_op_cbfunc_t opcbfunc;
     opal_pmix_modex_cbfunc_t mdxcbfunc;
     opal_pmix_value_cbfunc_t valcbfunc;

opal/mca/pmix/ext11/pmix_ext_server_south.c

@@ -1,7 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
- * Copyright (c) 2014-2016 Research Organization for Information Science
+ * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014      Mellanox Technologies, Inc.
@@ -193,7 +193,11 @@ static void opcbfunc(pmix_status_t status, void *cbdata)
     if (NULL != op->opcbfunc) {
         op->opcbfunc(pmix1_convert_rc(status), op->cbdata);
     }
-    OBJ_RELEASE(op);
+    if (op->active) {
+        op->active = false;
+    } else {
+        OBJ_RELEASE(op);
+    }
 }
 
 int pmix1_server_register_nspace(opal_jobid_t jobid,
@@ -207,7 +211,7 @@ int pmix1_server_register_nspace(opal_jobid_t jobid,
     size_t sz, szmap, m, n;
     char nspace[PMIX_MAX_NSLEN];
     pmix_status_t rc;
-    pmix1_opcaddy_t *op;
+    pmix1_opcaddy_t op;
     opal_list_t *pmapinfo;
     opal_pmix1_jobid_trkr_t *job;
 
@@ -253,15 +257,16 @@ int pmix1_server_register_nspace(opal_jobid_t jobid,
     }
 
     /* setup the caddy */
-    op = OBJ_NEW(pmix1_opcaddy_t);
-    op->info = pinfo;
-    op->sz = sz;
-    op->opcbfunc = cbfunc;
-    op->cbdata = cbdata;
+    OBJ_CONSTRUCT(&op, pmix1_opcaddy_t);
+    op.info = pinfo;
+    op.sz = sz;
+    op.opcbfunc = cbfunc;
+    op.cbdata = cbdata;
+    op.active = true;
     rc = PMIx_server_register_nspace(nspace, nlocalprocs, pinfo, sz,
-                                     opcbfunc, op);
-    if (PMIX_SUCCESS != rc) {
-        OBJ_RELEASE(op);
+                                     opcbfunc, &op);
+    if (PMIX_SUCCESS == rc) {
+        PMIX_WAIT_FOR_COMPLETION(op.active);
     }
     return pmix1_convert_rc(rc);
 }