From fe505c759b467f80129b24101f138aec21000b68 Mon Sep 17 00:00:00 2001 From: Weikuan Yu Date: Wed, 7 Jul 2004 00:06:18 +0000 Subject: [PATCH] 1) Done with the initilization test 2) To test the point-to-point This commit was SVN r1551. --- src/mca/ptl/elan/src/ptl_elan_frag.c | 4 +- src/mca/ptl/elan/src/ptl_elan_init.c | 10 ++++ src/mca/ptl/elan/src/ptl_elan_module.c | 6 +- src/mca/ptl/elan/tests/Makefile.am | 10 ++-- src/mca/ptl/elan/tests/mpi_test.c | 16 ++++++ src/mca/ptl/elan/tests/qsnet_init.c | 6 ++ src/mca/ptl/elan/tests/qsnet_qdma.c | 41 +++++-------- src/mca/ptl/elan/tests/qsnet_rdma.c | 79 +++++++++++++++++++++----- 8 files changed, 124 insertions(+), 48 deletions(-) create mode 100644 src/mca/ptl/elan/tests/mpi_test.c diff --git a/src/mca/ptl/elan/src/ptl_elan_frag.c b/src/mca/ptl/elan/src/ptl_elan_frag.c index c55881c40a..030f490de9 100644 --- a/src/mca/ptl/elan/src/ptl_elan_frag.c +++ b/src/mca/ptl/elan/src/ptl_elan_frag.c @@ -39,7 +39,7 @@ ompi_class_t mca_ptl_elan_send_frag_t_class = { static void mca_ptl_elan_recv_frag_construct (mca_ptl_elan_recv_frag_t * frag) { - OBJ_CONSTRUCT (frag, mca_ptl_elan_recv_frag_t); + /*OBJ_CONSTRUCT (frag, mca_ptl_elan_recv_frag_t);*/ frag->frag_hdr_cnt = 0; frag->frag_msg_cnt = 0; frag->frag.qdma = NULL; @@ -63,7 +63,7 @@ mca_ptl_elan_recv_frag_destruct (mca_ptl_elan_recv_frag_t * frag) free (frag->alloc_buff); frag->alloc_buff = NULL; frag->unex_buff = NULL; - OBJ_DESTRUCT (frag); + /*OBJ_DESTRUCT (frag);*/ } ompi_class_t mca_ptl_elan_recv_frag_t_class = { diff --git a/src/mca/ptl/elan/src/ptl_elan_init.c b/src/mca/ptl/elan/src/ptl_elan_init.c index ba87fe0976..843cf16291 100644 --- a/src/mca/ptl/elan/src/ptl_elan_init.c +++ b/src/mca/ptl/elan/src/ptl_elan_init.c @@ -176,6 +176,16 @@ ompi_elan_attach_network (mca_ptl_elan_state_t * ems) return OMPI_ERROR; } + /* Allocate more than we need to keep the heap in sync */ + ems->elan_localvps = (int *) malloc (sizeof (int) * (ems->elan_nvp + 1)); + + if (NULL == ems->elan_localvps) { + ompi_output (0, + "[%s:%d] error in malloc for elan_localvps \n", + __FILE__, __LINE__); + return OMPI_ERROR; + } + /* Set all to non local initially */ for (vp = 0; vp < ems->elan_nvp; vp++) ems->elan_localvps[vp] = -1; diff --git a/src/mca/ptl/elan/src/ptl_elan_module.c b/src/mca/ptl/elan/src/ptl_elan_module.c index 0684f56420..9e986789f4 100644 --- a/src/mca/ptl/elan/src/ptl_elan_module.c +++ b/src/mca/ptl/elan/src/ptl_elan_module.c @@ -26,6 +26,8 @@ #include "ptl_elan_req.h" #include "ptl_elan_priv.h" +#define UNIT_TESTING 1 + mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module = { { /* Base module information about itself */ @@ -232,12 +234,14 @@ mca_ptl_elan_module_init (int *num_ptls, return NULL; } +#ifndef UNIT_TESTING if (OMPI_SUCCESS != mca_ptl_elan_module_register(&mca_ptl_elan_module)) { ompi_output(0, - "[%s:%d] error in malloc for elan PTL references\n", + "[%s:%d] error in registering with Runtime/OOB \n", __FILE__, __LINE__); return NULL; } +#endif ptls = (mca_ptl_t **) malloc (elan_mp->elan_num_ptls * sizeof (mca_ptl_elan_t *)); diff --git a/src/mca/ptl/elan/tests/Makefile.am b/src/mca/ptl/elan/tests/Makefile.am index 7717932cb7..c6323025c1 100644 --- a/src/mca/ptl/elan/tests/Makefile.am +++ b/src/mca/ptl/elan/tests/Makefile.am @@ -12,19 +12,21 @@ AM_CPPFLAGS = -I$(top_ompi_builddir)/src/include \ LDFLAGS += -L$(prefix)/lib -L../src/.libs -EXECS = qsnet_init qsnet_qdma qsnet_rdma +EXECS = qsnet_init qsnet_qdma qsnet_rdma mpitest units: $(EXECS) qsnet_init: qsnet_init.c - $(CC) $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan + $(CC) -g $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan qsnet_qdma: qsnet_qdma.c - $(CC) $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan + $(CC) -g $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan qsnet_rdma: qsnet_rdma.c - $(CC) $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan + $(CC) -g $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan +mpitest: mpi_test.c + $(CC) -I${HOME}/installs/openmpi/include -L${HOME}/installs/openmpi/lib -o mpitest mpi_test.c -lmpi clean: -rm -rf $(EXECS) diff --git a/src/mca/ptl/elan/tests/mpi_test.c b/src/mca/ptl/elan/tests/mpi_test.c new file mode 100644 index 0000000000..a030d50cab --- /dev/null +++ b/src/mca/ptl/elan/tests/mpi_test.c @@ -0,0 +1,16 @@ +#include +#include "mpi.h" +#include "test_util.h" + +int main (int argc, char ** argv) +{ + int proc, nproc; + /* Get some environmental variables set for Open MPI, OOB */ + env_init_for_elan(); + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &proc); + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + MPI_Finalize(); + return 0; +} diff --git a/src/mca/ptl/elan/tests/qsnet_init.c b/src/mca/ptl/elan/tests/qsnet_init.c index 72342f477b..539e3cfd59 100644 --- a/src/mca/ptl/elan/tests/qsnet_init.c +++ b/src/mca/ptl/elan/tests/qsnet_init.c @@ -1,8 +1,11 @@ +#include #include "qsnet/fence.h" #include "ptl_elan.h" #include "ptl_elan_priv.h" +#include "test_util.h" extern mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module; + int main (int argc, char ** argv) { bool allow_threads; @@ -10,6 +13,9 @@ int main (int argc, char ** argv) int input; int num; + /* Get some environmental variables set for Open MPI, OOB */ + env_init_for_elan(); + /* Initialization test */ mca_ptl_elan_module_open(); mca_ptl_elan_module_init(&num, &allow_threads, &have_hidden_threads); diff --git a/src/mca/ptl/elan/tests/qsnet_qdma.c b/src/mca/ptl/elan/tests/qsnet_qdma.c index f15cc69401..af541bd0e1 100644 --- a/src/mca/ptl/elan/tests/qsnet_qdma.c +++ b/src/mca/ptl/elan/tests/qsnet_qdma.c @@ -1,39 +1,33 @@ #include "qsnet/fence.h" #include "ptl_elan.h" #include "ptl_elan_priv.h" +#include "test_util.h" extern mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module; -int test_qdma(mca_ptl_elan_module_1_0_0_t *emp, int reps); - -int ptl_elan_init_event(); -int ptl_elan_free_event(); -int ptl_elan_poll_event(); -int ptl_elan_wait_event(); - -int ptl_elan_queue_send(); /* Initialize a Tx event */ +int ptl_elan_queue_send(); /* Initialize a Tx event */ int ptl_elan_queue_recv(); /* Wait for a recv event */ +int test_qdma(mca_ptl_elan_module_1_0_0_t *emp, int reps); int main (int argc, char ** argv) { - /* Initialization test */ - ompi_mca_ptl_elan_init (&mca_ptl_elan_module); + bool allow_threads; + bool have_hidden_threads; + int input; + int num; - /* Please replace with a barrier at the beginning */ + /* Get some environmental variables set for Open MPI, OOB */ + env_init_for_elan(); - sleep(5); /* Sleep 5 seconds for others to catch up */ + /* Initialization */ + mca_ptl_elan_module_open(); + mca_ptl_elan_module_init(&num, &allow_threads, &have_hidden_threads); + mca_ptl_elan_module_control(1, &input, 4); + /* Testing QDMA */ test_qdma(&mca_ptl_elan_module, 1); - /* Please replace with a barrier at the end */ - sleep(5); /* Sleep 5 seconds for others to catch up */ - - /* Finalize the device */ - ompi_mca_ptl_elan_finalize (&mca_ptl_elan_module); - - /* Tell alive */ - fprintf(stdout, "I am still alive\n"); - fflush(stdout); + mca_ptl_elan_module_close(); return 0; } @@ -57,7 +51,6 @@ int test_qdma(mca_ptl_elan_module_1_0_0_t *emp, int reps) if (0 != ptl->elan_vp) { r--; - /*elan_queueRxWait(qr,rbuf,waitType);*/ ptl_elan_queue_recv(); } @@ -89,7 +82,3 @@ int ptl_elan_queue_recv () return OMPI_SUCCESS; } -int ptl_elan_init_event() -{ - return OMPI_SUCCESS; -} diff --git a/src/mca/ptl/elan/tests/qsnet_rdma.c b/src/mca/ptl/elan/tests/qsnet_rdma.c index 4228feede1..c748152e67 100644 --- a/src/mca/ptl/elan/tests/qsnet_rdma.c +++ b/src/mca/ptl/elan/tests/qsnet_rdma.c @@ -1,35 +1,84 @@ #include "qsnet/fence.h" #include "ptl_elan.h" #include "ptl_elan_priv.h" +#include "test_util.h" extern mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module; +int ptl_elan_rdma_put(); /* Initialize a Tx event */ +int ptl_elan_rdma_get(); /* Wait for a recv event */ int test_rdma(mca_ptl_elan_module_1_0_0_t *emp, int reps); int main (int argc, char ** argv) { - /* Initialization test */ - ompi_mca_ptl_elan_init (&mca_ptl_elan_module); + bool allow_threads; + bool have_hidden_threads; + int input; + int num; - /* Please replace with a barrier at the beginning */ - sleep(5); /* Sleep 5 seconds for others to catch up */ + /* Get some environmental variables set for Open MPI, OOB */ + env_init_for_elan(); + /* Initialization */ + mca_ptl_elan_module_open(); + mca_ptl_elan_module_init(&num, &allow_threads, &have_hidden_threads); + mca_ptl_elan_module_control(1, &input, 4); + + /* Testing QDMA */ test_rdma(&mca_ptl_elan_module, 1); - /* Please replace with a barrier at the end */ - sleep(5); /* Sleep 5 seconds for others to catch up */ - - /* Finalize the device */ - ompi_mca_ptl_elan_finalize (&mca_ptl_elan_module); - - /* Tell alive */ - fprintf(stdout, "I am still alive\n"); - fflush(stdout); + mca_ptl_elan_module_close(); return 0; } int test_rdma(mca_ptl_elan_module_1_0_0_t *emp, int reps) { - /* TODO: To test rdma (put/get) function */ - return 0; + uint64_t start0, end0; + double t; + int r; + double nsec; + + ELAN4_CTX *ctx; + RAIL *rail; + mca_ptl_elan_t *ptl; + + r = reps; + ptl = emp->elan_ptls[0]; + ctx = emp->elan_ctrl->elan_ctx; + rail = (RAIL *) emp->elan_ctrl->elan_rail[0]; + + start0 = elan4_clock(ctx); + + if (0 != ptl->elan_vp) { + r--; + ptl_elan_rdma_get(); + } + + while (--r >= 0) { + /* Trigger a send event */ + ptl_elan_rdma_put(); + ptl_elan_rdma_get(); + } + + if (0 != ptl->elan_vp) { + /* Trigger one more send */ + ptl_elan_rdma_put(); + } + + end0 = elan4_clock(ctx); + nsec = ((end0 - start0) / reps); + t = ((double) nsec)/(2*1000.0); + + return(0); } + +int ptl_elan_rdma_put() { + return OMPI_SUCCESS; +} + +/* This function needs no event related knowledge */ +int ptl_elan_rdma_get () +{ + return OMPI_SUCCESS; +} +