##
# Copyright 2019-2022 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for OpenMPI, implemented as an easyblock
@author: Kenneth Hoste (Ghent University)
@author: Robert Mijakovic (LuxProvide)
"""
import os
import re
from distutils.version import LooseVersion
import easybuild.tools.toolchain as toolchain
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
from easybuild.tools.modules import get_software_root
from easybuild.tools.systemtools import check_os_dependency, get_shared_lib_ext
from easybuild.tools.toolchain.mpi import get_mpi_cmd_template
class EB_OpenMPI(ConfigureMake):
"""OpenMPI easyblock."""
def configure_step(self):
"""Custom configuration step for OpenMPI."""
def config_opt_used(key, enable_opt=False):
"""Helper function to check whether a configure option is already specified in 'configopts'."""
if enable_opt:
regex = '--(disable|enable)-%s' % key
else:
regex = '--(with|without)-%s' % key
return bool(re.search(regex, self.cfg['configopts']))
config_opt_names = [
# suppress failure modes in relation to mpirun path
'mpirun-prefix-by-default',
# build shared libraries
'shared',
]
for key in config_opt_names:
if not config_opt_used(key, enable_opt=True):
self.cfg.update('configopts', '--enable-%s' % key)
# List of EasyBuild dependencies for which OMPI has known options
known_dependencies = ('CUDA', 'hwloc', 'libevent', 'libfabric', 'PMIx', 'UCX')
# Value to use for `--with-<dep>=<value>` if the dependency is not specified in the easyconfig
# No entry is interpreted as no option added at all
# This is to make builds reproducible even when the system libraries are changed and avoids failures
# due to e.g. finding only PMIx but not libevent on the system
unused_dep_value = dict()
# Known options since version 3.0 (no earlier ones checked)
if LooseVersion(self.version) >= LooseVersion('3.0'):
# Default to disable the option with "no"
unused_dep_value = {dep: 'no' for dep in known_dependencies}
# For these the default is to use an internal copy and not using any is not supported
for dep in ('hwloc', 'libevent', 'PMIx'):
unused_dep_value[dep] = 'internal'
# handle dependencies
for dep in known_dependencies:
opt_name = dep.lower()
# If the option is already used, don't add it
if config_opt_used(opt_name):
continue
# libfabric option renamed in OpenMPI 3.1.0 to ofi
if dep == 'libfabric' and LooseVersion(self.version) >= LooseVersion('3.1'):
opt_name = 'ofi'
# Check new option name. They are synonyms since 3.1.0 for backward compatibility
if config_opt_used(opt_name):
continue
dep_root = get_software_root(dep)
# If the dependency is loaded, specify its path, else use the "unused" value, if any
if dep_root:
opt_value = dep_root
else:
opt_value = unused_dep_value.get(dep)
if opt_value is not None:
self.cfg.update('configopts', '--with-%s=%s' % (opt_name, opt_value))
if bool(get_software_root('PMIx')) != bool(get_software_root('libevent')):
raise EasyBuildError('You must either use both PMIx and libevent as dependencies or none of them. '
'This is to enforce the same libevent is used for OpenMPI as for PMIx or '
'the behavior may be unpredictable.')
# check whether VERBS support should be enabled
if not config_opt_used('verbs'):
# for OpenMPI v4.x, the openib BTL should be disabled when UCX is used;
# this is required to avoid "error initializing an OpenFabrics device" warnings,
# see also https://www.open-mpi.org/faq/?category=all#ofa-device-error
is_ucx_enabled = ('--with-ucx' in self.cfg['configopts'] and
'--with-ucx=no' not in self.cfg['configopts'])
if LooseVersion(self.version) >= LooseVersion('4.0.0') and is_ucx_enabled:
verbs = False
else:
# auto-detect based on available OS packages
os_packages = EASYCONFIG_CONSTANTS['OS_PKG_IBVERBS_DEV'][0]
verbs = any(check_os_dependency(osdep) for osdep in os_packages)
# for OpenMPI v5.x, the verbs support is removed, only UCX is available
# see https://github.com/open-mpi/ompi/pull/6270
if LooseVersion(self.version) < LooseVersion('5.0.0'):
if verbs:
self.cfg.update('configopts', '--with-verbs')
else:
self.cfg.update('configopts', '--without-verbs')
super(EB_OpenMPI, self).configure_step()
def test_step(self):
"""Test step for OpenMPI"""
# Default to `make check` if nothing is set. Disable with "runtest = False" in the EC
if self.cfg['runtest'] is None:
self.cfg['runtest'] = 'check'
super(EB_OpenMPI, self).test_step()
def load_module(self, *args, **kwargs):
"""
Load (temporary) module file, after resetting to initial environment.
Also put RPATH wrappers back in place if needed, to ensure that sanity check commands work as expected.
"""
super(EB_OpenMPI, self).load_module(*args, **kwargs)
# ensure RPATH wrappers are in place, otherwise compiling minimal test programs will fail
if build_option('rpath'):
if self.toolchain.options.get('rpath', True):
self.toolchain.prepare_rpath_wrappers(rpath_filter_dirs=self.rpath_filter_dirs,
rpath_include_dirs=self.rpath_include_dirs)
def sanity_check_step(self):
"""Custom sanity check for OpenMPI."""
bin_names = ['mpicc', 'mpicxx', 'mpif90', 'mpifort', 'ompi_info', 'opal_wrapper']
if LooseVersion(self.version) >= LooseVersion('5.0.0'):
bin_names.append('prterun')
else:
if '--with-orte=no' not in self.cfg['configopts'] and '--without-orte' not in self.cfg['configopts']:
bin_names.extend(['orterun', 'mpirun'])
bin_files = [os.path.join('bin', x) for x in bin_names]
shlib_ext = get_shared_lib_ext()
lib_names = ['mpi_mpifh', 'mpi', 'open-pal']
if LooseVersion(self.version) >= LooseVersion('5.0.0'):
lib_names.append('prrte')
else:
lib_names.extend(['ompitrace', 'open-rte'])
lib_files = [os.path.join('lib', 'lib%s.%s' % (x, shlib_ext)) for x in lib_names]
inc_names = ['mpi-ext', 'mpif-config', 'mpif', 'mpi', 'mpi_portable_platform']
if LooseVersion(self.version) >= LooseVersion('5.0.0'):
inc_names.append('prte')
inc_files = [os.path.join('include', x + '.h') for x in inc_names]
custom_paths = {
'files': bin_files + inc_files + lib_files,
'dirs': [],
}
# make sure MPI compiler wrappers pick up correct compilers
expected = {
'mpicc': os.getenv('CC', 'gcc'),
'mpicxx': os.getenv('CXX', 'g++'),
'mpifort': os.getenv('FC', 'gfortran'),
'mpif90': os.getenv('F90', 'gfortran'),
}
# actual pattern for gfortran is "GNU Fortran"
for key in ['mpifort', 'mpif90']:
if expected[key] == 'gfortran':
expected[key] = "GNU Fortran"
# for PGI, correct pattern is "pgfortran" with mpif90
if expected['mpif90'] == 'pgf90':
expected['mpif90'] = 'pgfortran'
custom_commands = ["%s --version | grep '%s'" % (key, expected[key]) for key in sorted(expected.keys())]
# Add minimal test program to sanity checks
# Run with correct MPI launcher
mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.OPENMPI, dict(), mpi_version=self.version)
# Limit number of ranks to 8 to avoid it failing due to hyperthreading
ranks = min(8, self.cfg['parallel'])
for src, compiler in (('hello_c.c', 'mpicc'), ('hello_mpifh.f', 'mpifort'), ('hello_usempi.f90', 'mpif90')):
src_path = os.path.join(self.cfg['start_dir'], 'examples', src)
if os.path.exists(src_path):
test_exe = os.path.join(self.builddir, 'mpi_test_' + os.path.splitext(src)[0])
self.log.info("Adding minimal MPI test program to sanity checks: %s", test_exe)
# Build test binary
custom_commands.append("%s %s -o %s" % (compiler, src_path, test_exe))
# Run the test if chosen
if build_option('mpi_tests'):
params.update({'nr_ranks': ranks, 'cmd': test_exe})
# Allow oversubscription for this test (in case of hyperthreading)
custom_commands.append("OMPI_MCA_rmaps_base_oversubscribe=1 " + mpi_cmd_tmpl % params)
# Run with 1 process which may trigger other bugs
# See https://github.com/easybuilders/easybuild-easyconfigs/issues/12978
params['nr_ranks'] = 1
custom_commands.append(mpi_cmd_tmpl % params)
super(EB_OpenMPI, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)