From 6da16ab0d7c7685cd8c34a7ff2d9f863a6979c4f Mon Sep 17 00:00:00 2001 From: Terry Dontje Date: Thu, 16 Dec 2010 15:11:17 +0000 Subject: [PATCH] add format parameter and layout format to OMPI_Affinity_str This commit was SVN r24182. --- README | 18 +++ ompi/mpiext/affinity/OMPI_Affinity_str.3in | 116 ++++++++++++++++- ompi/mpiext/affinity/README.txt | 8 +- ompi/mpiext/affinity/c/affinity_str.c | 121 ++++++++++++++++-- ompi/mpiext/affinity/mpiext_affinity_c.h | 10 +- opal/mca/paffinity/base/base.h | 7 + .../paffinity/base/paffinity_base_service.c | 56 ++++++++ 7 files changed, 318 insertions(+), 18 deletions(-) diff --git a/README b/README index c122d64b4b..c511d33dc0 100644 --- a/README +++ b/README @@ -561,6 +561,24 @@ Network Support See http://runtime.bordeaux.inria.fr/knem/ for details on Knem. +Open MPI Extensions +------------------- + +- Extensions framework added. See the "Open MPI API Extensions" + section below for more information on compiling and using + extensions. + +- The following extensions are included in this version of Open MPI: + + - affinity: Provides the OMPI_Affinity_str() routine on retrieving + a string that contains what resources a process is bound to. See + its man page for more details. + - cr: Provides routines to access to checkpoint restart routines. + See ompi/mpiext/cr/mpiext_cr_c.h for a listing of availble + functions. + - example: A non-functional extension; its only purpose is to + provide an example for how to create other extensions. + =========================================================================== Building Open MPI diff --git a/ompi/mpiext/affinity/OMPI_Affinity_str.3in b/ompi/mpiext/affinity/OMPI_Affinity_str.3in index 697d385b74..a5cbc748dd 100644 --- a/ompi/mpiext/affinity/OMPI_Affinity_str.3in +++ b/ompi/mpiext/affinity/OMPI_Affinity_str.3in @@ -1,4 +1,4 @@ -.\" Copyright 2007-2008 Sun Microsystems, Inc. +.\" Copyright 2007-2010 Oracle and/or its affiliates. All rights reserved. .\" Copyright (c) 1996 Thinking Machines Corporation .\" Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. .TH OMPI_Affinity_str 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#" @@ -12,7 +12,8 @@ #include #include -int OMPI_Affinity_str(char \fIompi_bound\fP[OMPI_AFFINITY_STRING_MAX], +int OMPI_Affinity_str(ompi_affinity_fmt_type_t \fIfmt_type\fP, + char \fIompi_bound\fP[OMPI_AFFINITY_STRING_MAX], char \fIcurrent_binding\fP[OMPI_AFFINITY_STRING_MAX], char \fIexists\fP[OMPI_AFFINITY_STRING_MAX]) .fi @@ -21,6 +22,22 @@ There is no Fortran binding for this function. . .SH C++ Syntax There is no C++ binding for this function. +. +.SH INPUT PARAMETERS +.ft R +.TP 1i +fmt_type +An enum indicating how to format the returned ompi_bound and +current_binding strings. OMPI_AFFINITY_RSRC_STRING_FMT returns the +string as human-readable resource names, such as "socket 0, core 0". + +OMPI_AFFINITY_LAYOUT_FMT returns ASCII art representing where this MPI +process is bound relative to the machine resource layout. For example +"[. B][. .]" shows the process that called the routine is bound to +socket 0, core 1 in a system with 2 sockets, each containing 2 cores. + +See below for more output examples. + . .SH OUTPUT PARAMETERS .ft R @@ -80,7 +97,100 @@ that this process can see (which is .I usually all processors in the system). -.SH SEE ALSO +.SH Examples +.ft R +\fBExample 1:\fP Print out processes binding using resource string format. +.sp +.nf + int rank; + char ompi_bound[OMPI_AFFINITY_STRING_MAX]; + char current_binding[OMPI_AFFINITY_STRING_MAX]; + char exists[OMPI_AFFINITY_STRING_MAX]; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + OMPI_Affinity_str(OMPI_AFFINITY_RSRC_STRING_FMT, + ompi_bound, current_binding, exists); + printf("rank %d: \\n" + " ompi_bound: %s\\n" + " current_binding: %s\\n" + " exists: %s\\n", + rank, ompi_bound, current_binding, exists); + ... +.fi +.PP +Output of mpirun -np 2 -bind-to-core a.out: +.nf +rank 0: + ompi_bound: socket 0[core 0] + current_binding: socket 0[core 0] + exists: socket 0 has 4 cores +rank 1: + ompi_bound: socket 0[core 1] + current_binding: socket 0[core 1] + exists: socket 0 has 4 cores +.fi +.PP +Output of mpirun -np 2 -bind-to-socket a.out: +.nf +rank 0: + ompi_bound: socket 0[core 0-3] + current_binding: Not bound (or bound to all available processors) + exists: socket 0 has 4 cores +rank 1: + ompi_bound: socket 0[core 0-3] + current_binding: Not bound (or bound to all available processors) + exists: socket 0 has 4 cores +.fi +.sp +.br +\fBExample 2:\fP Print out processes binding using layout string format. +.sp +.nf + int rank; + char ompi_bound[OMPI_AFFINITY_STRING_MAX]; + char current_binding[OMPI_AFFINITY_STRING_MAX]; + char exists[OMPI_AFFINITY_STRING_MAX]; + + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + + OMPI_Affinity_str(OMPI_AFFINITY_LAYOUT_FMT, + ompi_bound, current_binding, exists); + printf("rank %d: \\n" + " ompi_bound: %s\\n" + " current_binding: %s\\n" + " exists: %s\\n", + rank, ompi_bound, current_binding, exists); + ... +.fi +.PP +Output of mpirun -np 2 -bind-to-core a.out: +.nf +rank 0: + ompi_bound: [B . . .] + current_binding: [B . . .] + exists: [. . . .] +rank 1: + ompi_bound: [. B . .] + current_binding: [. B . .] + exists: [. . . .] +.fi +.PP +Output of mpirun -np 2 -bind-to-socket a.out: +.nf +rank 0: + ompi_bound: [B B B B] + current_binding: [B B B B] + exists: [. . . .] +rank 1: + ompi_bound: [B B B B] + current_binding: [B B B B] + exists: [. . . .] +.fi + +.SH See Also .ft R .nf mpirun(1) diff --git a/ompi/mpiext/affinity/README.txt b/ompi/mpiext/affinity/README.txt index 0c372ff0a7..f22970a331 100644 --- a/ompi/mpiext/affinity/README.txt +++ b/ompi/mpiext/affinity/README.txt @@ -1,10 +1,16 @@ Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. Jeff Squyres 19 April 2010 +Terry Dontje +18 November 2010 This extension provides a single new function, OMPI_Affinity_str(), -that provides 3 prettyprint strings as output: +that takes a format value and then provides 3 prettyprint strings as output: + +fmt_type: is an enum that tells OMPI_Affinity_str() whether to use a resource description +string or layout string format for ompi_bound and currently_bound output strings. ompi_bound: describes what sockets/cores Open MPI bound this process to (or indicates that Open MPI did not bind this process). diff --git a/ompi/mpiext/affinity/c/affinity_str.c b/ompi/mpiext/affinity/c/affinity_str.c index 803c0ed3d6..2b8d7ca5f1 100644 --- a/ompi/mpiext/affinity/c/affinity_str.c +++ b/ompi/mpiext/affinity/c/affinity_str.c @@ -3,6 +3,7 @@ * University Research and Technology * Corporation. All rights reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -32,12 +33,15 @@ static const char FUNC_NAME[] = "OMPI_Affinity"; -static int fill_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]); -static int fill_current_binding(char str[OMPI_AFFINITY_STRING_MAX]); -static int fill_exists(char str[OMPI_AFFINITY_STRING_MAX]); +static int get_rsrc_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]); +static int get_rsrc_current_binding(char str[OMPI_AFFINITY_STRING_MAX]); +static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX]); +static int get_layout_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]); +static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX]); +static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX]); - -int OMPI_Affinity_str(char ompi_bound[OMPI_AFFINITY_STRING_MAX], +int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type, + char ompi_bound[OMPI_AFFINITY_STRING_MAX], char current_binding[OMPI_AFFINITY_STRING_MAX], char exists[OMPI_AFFINITY_STRING_MAX]) { @@ -46,16 +50,29 @@ int OMPI_Affinity_str(char ompi_bound[OMPI_AFFINITY_STRING_MAX], memset(ompi_bound, 0, sizeof(ompi_bound)); memset(current_binding, 0, sizeof(current_binding)); - if (OPAL_SUCCESS != (ret = fill_ompi_bound(ompi_bound)) || - OPAL_SUCCESS != (ret = fill_current_binding(current_binding)) || - OPAL_SUCCESS != (ret = fill_exists(exists))) { - return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret, FUNC_NAME); + switch(fmt_type) { + case OMPI_AFFINITY_RSRC_STRING_FMT: + if (OPAL_SUCCESS != (ret = get_rsrc_ompi_bound(ompi_bound)) || + OPAL_SUCCESS != (ret = get_rsrc_current_binding(current_binding)) || + OPAL_SUCCESS != (ret = get_rsrc_exists(exists))) { + return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret, FUNC_NAME); + } + break; + case OMPI_AFFINITY_LAYOUT_FMT: + if (OPAL_SUCCESS != (ret = get_layout_ompi_bound(ompi_bound)) || + OPAL_SUCCESS != (ret = get_layout_current_binding(current_binding)) || + OPAL_SUCCESS != (ret = get_layout_exists(exists))) { + return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, ret, FUNC_NAME); + } + break; + default: + return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG, FUNC_NAME); } return MPI_SUCCESS; } -static int fill_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]) +static int get_rsrc_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]) { int ret; opal_paffinity_base_cpu_set_t cset; @@ -78,7 +95,7 @@ static int fill_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]) return opal_paffinity_base_cset2str(str, OMPI_AFFINITY_STRING_MAX, &cset); } -static int fill_current_binding(char str[OMPI_AFFINITY_STRING_MAX]) +static int get_rsrc_current_binding(char str[OMPI_AFFINITY_STRING_MAX]) { int ret, flag; opal_paffinity_base_cpu_set_t cset; @@ -102,7 +119,7 @@ static int fill_current_binding(char str[OMPI_AFFINITY_STRING_MAX]) /* Prettyprint a list of all available processors */ -static int fill_exists(char str[OMPI_AFFINITY_STRING_MAX]) +static int get_rsrc_exists(char str[OMPI_AFFINITY_STRING_MAX]) { int ret, i, num_sockets, num_cores; char tmp[BUFSIZ]; @@ -137,3 +154,83 @@ static int fill_exists(char str[OMPI_AFFINITY_STRING_MAX]) return OPAL_SUCCESS; } + +static int get_layout_ompi_bound(char str[OMPI_AFFINITY_STRING_MAX]) +{ + int ret; + opal_paffinity_base_cpu_set_t cset; + + /* If OMPI did not bind, indicate that */ + if (!opal_paffinity_base_bound) { + const char tmp[] = "Open MPI did not bind this process"; + strncpy(str, tmp, OMPI_AFFINITY_STRING_MAX - 1); + return OPAL_SUCCESS; + } + + /* Find out what OMPI bound us to and prettyprint it */ + ret = + opal_paffinity_base_parse_binding(opal_paffinity_base_applied_binding, + &cset); + if (OPAL_SUCCESS != ret) { + return ret; + } + + return opal_paffinity_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, &cset); +} + +static int get_layout_current_binding(char str[OMPI_AFFINITY_STRING_MAX]) +{ + int ret = OPAL_SUCCESS, flag; + opal_paffinity_base_cpu_set_t cset; + + /* Get our binding */ + ret = opal_paffinity_base_get(&cset); + if (OPAL_SUCCESS != ret) { + return ret; + } + + /* Are we bound anywhere? */ + OPAL_PAFFINITY_PROCESS_IS_BOUND(cset, &flag); + if (!flag) { + const char tmp[] = "Not bound (or bound to all available processors)"; + strncat(str, tmp, OMPI_AFFINITY_STRING_MAX - 1); + return OPAL_SUCCESS; + } + + return opal_paffinity_base_cset2mapstr(str, OMPI_AFFINITY_STRING_MAX, &cset); +} + +/* Prettyprint a list of all available processors in layout format*/ +static int get_layout_exists(char str[OMPI_AFFINITY_STRING_MAX]) +{ + int ret, i, j, num_sockets, num_cores; + int len = OMPI_AFFINITY_STRING_MAX; + + str[0] = '\0'; + + /* Loop over the number of sockets in this machine */ + ret = opal_paffinity_base_get_socket_info(&num_sockets); + if (OPAL_SUCCESS != ret) { + return ret; + } + for (i = 0; i < num_sockets; ++i) { + strncat(str, "[", len - strlen(str)); + /* Loop over the number of cores in this socket */ + ret = opal_paffinity_base_get_core_info(i, &num_cores); + if (OPAL_SUCCESS != ret) { + return ret; + } + for (j = 0; j < num_cores; j++) { + if (0 < j) { + /* add space after first core is printed */ + strncat(str, " ", len - strlen(str)); + } + + /* mark core exists */ + strncat(str, ".", len - strlen(str)); + } + strncat(str, "]", len - strlen(str)); + } + + return OPAL_SUCCESS; +} diff --git a/ompi/mpiext/affinity/mpiext_affinity_c.h b/ompi/mpiext/affinity/mpiext_affinity_c.h index ca4c74791c..cec3da2c7e 100644 --- a/ompi/mpiext/affinity/mpiext_affinity_c.h +++ b/ompi/mpiext/affinity/mpiext_affinity_c.h @@ -2,6 +2,7 @@ * Copyright (c) 2004-2009 The Trustees of Indiana University. * All rights reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -12,7 +13,12 @@ #define OMPI_AFFINITY_STRING_MAX 1024 -OMPI_DECLSPEC int OMPI_Affinity_str(char ompi_bound[OMPI_AFFINITY_STRING_MAX], +typedef enum ompi_affinity_fmt { + OMPI_AFFINITY_RSRC_STRING_FMT, + OMPI_AFFINITY_LAYOUT_FMT +} ompi_affinity_fmt_t; + +OMPI_DECLSPEC int OMPI_Affinity_str(ompi_affinity_fmt_t fmt_type, + char ompi_bound[OMPI_AFFINITY_STRING_MAX], char current_binding[OMPI_AFFINITY_STRING_MAX], char exists[OMPI_AFFINITY_STRING_MAX]); - diff --git a/opal/mca/paffinity/base/base.h b/opal/mca/paffinity/base/base.h index f7f34cf29f..b71fd4d665 100644 --- a/opal/mca/paffinity/base/base.h +++ b/opal/mca/paffinity/base/base.h @@ -10,6 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -276,6 +277,12 @@ OPAL_DECLSPEC int opal_paffinity_base_slot_list_set(long rank, char *slot_str, OPAL_DECLSPEC int opal_paffinity_base_cset2str(char *str, int len, opal_paffinity_base_cpu_set_t *cset); +/** + * Make a prettyprint string for a cset with a map format. + */ +OPAL_DECLSPEC int opal_paffinity_base_cset2mapstr(char *str, int len, + opal_paffinity_base_cpu_set_t *cset); + /** * Debugging output stream */ diff --git a/opal/mca/paffinity/base/paffinity_base_service.c b/opal/mca/paffinity/base/paffinity_base_service.c index 377ed1aa70..f33407e262 100644 --- a/opal/mca/paffinity/base/paffinity_base_service.c +++ b/opal/mca/paffinity/base/paffinity_base_service.c @@ -11,6 +11,7 @@ * All rights reserved. * Copyright (c) 2007-2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2008 Voltaire. All rights reserved + * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. * * $COPYRIGHT$ * @@ -607,4 +608,59 @@ int opal_paffinity_base_cset2str(char *str, int len, return OPAL_SUCCESS; } +/** + * Make a prettyprint string for a cset in a map format. + * Example: [B_/__] + * Key: [] - signifies socket + * / - signifies core + * _ - signifies thread a process not bound to + * B - signifies thread a process is bound to + */ +int opal_paffinity_base_cset2mapstr(char *str, int len, + opal_paffinity_base_cpu_set_t *cset) +{ + int ret, i, j, k, m, num_sockets, num_cores, flag, count, + range_first=0, range_last; + char tmp[BUFSIZ]; + const int stmp = sizeof(tmp) - 1; + + str[0] = tmp[stmp] = '\0'; + + /* Loop over the number of sockets in this machine */ + ret = opal_paffinity_base_get_socket_info(&num_sockets); + if (OPAL_SUCCESS != ret) { + return ret; + } + for (i = 0; i < num_sockets; ++i) { + strncat(str, "[", len - strlen(str)); + /* Loop over the number of cores in this socket */ + ret = opal_paffinity_base_get_core_info(i, &num_cores); + if (OPAL_SUCCESS != ret) { + return ret; + } + for (j = 0; j < num_cores; j++) { + if (0 < j) { + /* add space after first core is printed */ + strncat(str, " ", len - strlen(str)); + } + + ret = opal_paffinity_base_get_map_to_processor_id(i, j, &k); + if (OPAL_SUCCESS != ret) { + return ret; + } + + flag = OPAL_PAFFINITY_CPU_ISSET(k, *cset); + if (flag) { + /* mark core as bound to process */ + strncat(str, "B", len - strlen(str)); + } else { + /* mark core as no process bound to it */ + strncat(str, ".", len - strlen(str)); + } + } + strncat(str, "]", len - strlen(str)); + } + + return OPAL_SUCCESS; +}