From c3beb0472e7f4cef69884384bdf435249ad82116 Mon Sep 17 00:00:00 2001 From: Mike Dubman Date: Thu, 21 Aug 2014 12:12:43 +0000 Subject: [PATCH] openib/btl: better detect max reg memory. OFED has no runtime versioning API :( based on http://www.open-mpi.org/community/lists/users/2014/08/25048.php reviewed by AlexM cmr=v1.8.2:reviewer=ompi-rm1.8 This commit was SVN r32569. --- opal/mca/btl/openib/btl_openib.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/opal/mca/btl/openib/btl_openib.c b/opal/mca/btl/openib/btl_openib.c index 1837713e7f..3f7a931bc4 100644 --- a/opal/mca/btl/openib/btl_openib.c +++ b/opal/mca/btl/openib/btl_openib.c @@ -620,6 +620,7 @@ static uint64_t calculate_total_mem (void) #endif } + static uint64_t calculate_max_reg (void) { struct stat statinfo; @@ -630,11 +631,7 @@ static uint64_t calculate_max_reg (void) mem_total = calculate_total_mem (); - if (0 == stat("/sys/module/mlx5_core", &statinfo)) { - /* mlx5 means that we have ofed 2.0 and it can always register 2xmem_total for any mlx hca */ - max_reg = 2 * mem_total; - } - else if (0 == stat("/sys/module/mlx4_core/parameters", &statinfo)) { + if (0 == stat("/sys/module/mlx4_core/parameters/log_num_mtt", &statinfo)) { mtts_per_seg = 1 << read_module_param("/sys/module/mlx4_core/parameters/log_mtts_per_seg", 1); num_mtt = 1 << read_module_param("/sys/module/mlx4_core/parameters/log_num_mtt", 20); if (1 == num_mtt) { @@ -643,12 +640,22 @@ static uint64_t calculate_max_reg (void) } max_reg = (num_mtt - reserved_mtt) * opal_getpagesize () * mtts_per_seg; - } else if (0 == stat("/sys/module/ib_mthca/parameters", &statinfo)) { + + } else if (0 == stat("/sys/module/ib_mthca/parameters/num_mtt", &statinfo)) { mtts_per_seg = 1 << read_module_param("/sys/module/ib_mthca/parameters/log_mtts_per_seg", 1); num_mtt = read_module_param("/sys/module/ib_mthca/parameters/num_mtt", 1 << 20); reserved_mtt = read_module_param("/sys/module/ib_mthca/parameters/fmr_reserved_mtts", 0); max_reg = (num_mtt - reserved_mtt) * opal_getpagesize () * mtts_per_seg; + + } else if ( + (0 == stat("/sys/module/mlx5_core", &statinfo)) || + (0 == stat("/sys/module/mlx4_core/parameters", &statinfo)) || + (0 == stat("/sys/module/ib_mthca/parameters", &statinfo)) + ) { + /* mlx5 means that we have ofed 2.0 and it can always register 2xmem_total for any mlx hca */ + max_reg = 2 * mem_total; + } else { /* Need to update to determine the registration limit for this configuration */