Skip to content
Snippets Groups Projects
Commit e8d09b61 authored by Damian Alvarez's avatar Damian Alvarez
Browse files

To allow LWP to have its own configuration modules

parent e8d2b5bd
Branches
No related tags found
No related merge requests found
...@@ -19,7 +19,7 @@ CORE = 'Core' ...@@ -19,7 +19,7 @@ CORE = 'Core'
COMPILER = 'Compiler' COMPILER = 'Compiler'
MPI = 'MPI' MPI = 'MPI'
MPI_SETTINGS = 'MPI_settings' MPI_SETTINGS = 'MPI_settings'
COMM_SETTINGS = 'comm_settings' PKG_SETTINGS = 'pkg_settings'
MODULECLASS_COMPILER = 'compiler' MODULECLASS_COMPILER = 'compiler'
MODULECLASS_MPI = 'mpi' MODULECLASS_MPI = 'mpi'
...@@ -60,7 +60,7 @@ mpi_relevant_versions = { ...@@ -60,7 +60,7 @@ mpi_relevant_versions = {
mpi_with_settings = ['psmpi', 'impi', 'OpenMPI', 'BullMPI'] mpi_with_settings = ['psmpi', 'impi', 'OpenMPI', 'BullMPI']
# Communication packages with settings modules # Communication packages with settings modules
comm_pkg_with_settings = ['UCX', 'NCCL'] pkg_with_settings = ['UCX', 'NCCL', 'LWP']
class FlexibleCustomHierarchicalMNS(HierarchicalMNS): class FlexibleCustomHierarchicalMNS(HierarchicalMNS):
"""Class implementing an example hierarchical module naming scheme.""" """Class implementing an example hierarchical module naming scheme."""
...@@ -79,6 +79,9 @@ class FlexibleCustomHierarchicalMNS(HierarchicalMNS): ...@@ -79,6 +79,9 @@ class FlexibleCustomHierarchicalMNS(HierarchicalMNS):
modname_regex = re.compile('^%s/\S+$' % re.escape('IntelMPI')) modname_regex = re.compile('^%s/\S+$' % re.escape('IntelMPI'))
elif name in ['-'.join([x, 'settings']) for x in mpi_with_settings]: elif name in ['-'.join([x, 'settings']) for x in mpi_with_settings]:
modname_regex = re.compile('^%s/\S+$' % re.escape('mpi-settings')) modname_regex = re.compile('^%s/\S+$' % re.escape('mpi-settings'))
elif name == 'LWP-settings':
# Match almost anything, since the name depends actually on the version, to avoid load conflicts
modname_regex = re.compile('^LWP-\S+/enable$')
else: else:
modname_regex = re.compile('^%s/\S+$' % re.escape(name)) modname_regex = re.compile('^%s/\S+$' % re.escape(name))
res = bool(modname_regex.match(short_modname)) res = bool(modname_regex.match(short_modname))
...@@ -172,12 +175,12 @@ class FlexibleCustomHierarchicalMNS(HierarchicalMNS): ...@@ -172,12 +175,12 @@ class FlexibleCustomHierarchicalMNS(HierarchicalMNS):
# no compiler in toolchain, dummy toolchain => Core module # no compiler in toolchain, dummy toolchain => Core module
subdir = CORE subdir = CORE
# except if the module is a MPI settings module # except if the module is a MPI settings module
stripped_name = re.sub('-settings$', '', ec['name']) stripped_name = ec['name'].split('-settings')[0]
if stripped_name in mpi_with_settings: if stripped_name in mpi_with_settings:
subdir = os.path.join(MPI_SETTINGS, stripped_name, ec['version']) subdir = os.path.join(MPI_SETTINGS, stripped_name, ec['version'])
# or a module is for a communicaiton packages with settings # or a module is for a package with settings
elif stripped_name in comm_pkg_with_settings and '-settings' in ec['name']: elif (stripped_name in pkg_with_settings and '-settings' in ec['name']):
subdir = os.path.join(COMM_SETTINGS, stripped_name) subdir = os.path.join(PKG_SETTINGS, stripped_name)
else: else:
tc_comp_name, tc_comp_ver = self._find_relevant_compiler_info(tc_comp_info) tc_comp_name, tc_comp_ver = self._find_relevant_compiler_info(tc_comp_info)
tc_mpi = det_toolchain_mpi(ec) tc_mpi = det_toolchain_mpi(ec)
...@@ -201,11 +204,13 @@ class FlexibleCustomHierarchicalMNS(HierarchicalMNS): ...@@ -201,11 +204,13 @@ class FlexibleCustomHierarchicalMNS(HierarchicalMNS):
""" """
Determine short module name, i.e. the name under which modules will be exposed to users. Determine short module name, i.e. the name under which modules will be exposed to users.
Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5 Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5
UCX-UD (for MPI settings) mpi-settings/plain, etc
""" """
stripped_name = re.sub('-settings$', '', ec['name']) stripped_name = re.sub('-settings$', '', ec['name'])
if stripped_name in mpi_with_settings and '-settings' in ec['name']: if stripped_name in mpi_with_settings and '-settings' in ec['name']:
return os.path.join('mpi-settings', ec['versionsuffix']) return os.path.join('mpi-settings', ec['versionsuffix'])
elif stripped_name.startswith('LWP') and '-settings' in ec['name']:
return os.path.join(ec['version'], 'enable')
else: else:
return super(FlexibleCustomHierarchicalMNS, self).det_short_module_name(ec) return super(FlexibleCustomHierarchicalMNS, self).det_short_module_name(ec)
...@@ -279,7 +284,7 @@ class FlexibleCustomHierarchicalMNS(HierarchicalMNS): ...@@ -279,7 +284,7 @@ class FlexibleCustomHierarchicalMNS(HierarchicalMNS):
if ec['name'] in mpi_with_settings: if ec['name'] in mpi_with_settings:
paths.append(os.path.join(MPI_SETTINGS, mpi_name, mpi_ver)) paths.append(os.path.join(MPI_SETTINGS, mpi_name, mpi_ver))
elif ec['name'] in ['UCX', 'NCCL']: elif ec['name'] in pkg_with_settings:
paths.append(os.path.join(COMM_SETTINGS, ec['name'])) paths.append(os.path.join(PKG_SETTINGS, ec['name']))
return paths return paths
easyblock = 'Rpm'
name = 'LWP-MPI'
version = '1.7.0'
local_lwp_timestamp = '20210730182932'
local_lwp_lib_name = 'libatos-lwp-mpi-openmpi-gcc.so'
homepage = ''
description = 'Loads the atos-lwp-mpi-openmpi-gcc 1.7.0 environment'
toolchain = {'name': 'gompi', 'version': '2021b'}
sources = [
f'atos-lwp-mpi-openmpi-gcc-{version}-Atos.{local_lwp_timestamp}.el8.x86_64.rpm',
]
builddependencies = [
('rpmrebuild', '2.16', '', SYSTEM)
]
dependencies = [
('LWP', '1.1.3', '', SYSTEM),
]
modextravars = {
'LWP_MPI': f'%(installdir)s/lib/{local_lwp_lib_name}',
'atos_lwp_module_mpi': '1',
}
modextrapaths = {
'LWP_LIBS': f'lib/{local_lwp_lib_name}'
}
local_path_to_fix = f'/opt/tools/profilers/atos-lwp/atos-lwp-mpi-openmpi-gcc'
sanity_check_paths = {
'dirs': ['lib', 'share'],
'files': [f'lib/{local_lwp_lib_name}']
}
postinstallcmds = [
# Move to bin/ lib64/ include/ libexec/ and share/ to the correct directory
f'mv %(installdir)s{local_path_to_fix}/* %(installdir)s',
# Clean the unneded directories
f'rm -Rf %(installdir)s/usr',
f'rm -Rf %(installdir)s/opt',
f'rm -Rf %(installdir)s/rpm',
]
moduleclass = 'tools'
easyblock = 'Rpm'
name = 'LWP-MPI'
version = '1.7.0'
local_lwp_timestamp = '20210730183006'
local_lwp_lib_name = 'libatos-lwp-mpi-intelmpi-icc.so'
homepage = ''
description = 'Loads the atos-lwp-mpi-intelmpi-icc 1.7.0 environment'
toolchain = {'name': 'impi', 'version': '2021b'}
sources = [
f'atos-lwp-mpi-intelmpi-icc-{version}-Atos.{local_lwp_timestamp}.el8.x86_64.rpm',
]
builddependencies = [
('rpmrebuild', '2.16', '', SYSTEM)
]
dependencies = [
('LWP', '1.1.3', '', SYSTEM),
]
modextravars = {
'LWP_MPI': f'%(installdir)s/lib/{local_lwp_lib_name}',
'atos_lwp_module_mpi': '1',
}
modextrapaths = {
'LWP_LIBS': f'lib/{local_lwp_lib_name}'
}
local_path_to_fix = f'/opt/tools/profilers/atos-lwp/atos-lwp-mpi-intelmpi-icc'
sanity_check_paths = {
'dirs': ['lib', 'share'],
'files': [f'lib/{local_lwp_lib_name}']
}
postinstallcmds = [
# Move to bin/ lib64/ include/ libexec/ and share/ to the correct directory
f'mv %(installdir)s{local_path_to_fix}/* %(installdir)s',
# Clean the unneded directories
f'rm -Rf %(installdir)s/usr',
f'rm -Rf %(installdir)s/opt',
f'rm -Rf %(installdir)s/rpm',
]
moduleclass = 'tools'
easyblock = 'SystemBundle'
name = 'LWP-settings'
version = 'LWP-omp'
homepage = ''
description = 'Loads the LWP-omp module for LWP'
toolchain = SYSTEM
modextravars = {
'atos_lwp_module_omp': '1',
'LWP_MODULE_OMPT_VERBOSE_LEVEL': '0',
}
# Ugly hack, since otherwise we can't add paths from the main installation without
# writting an easyblock
modluafooter = '''
prepend_path("LWP_LIBS",pathJoin(os.getenv("EBROOTLWP"),"lib64/liblwp_omp_gather.so"))
setenv("LWP_OMP","pathJoin(os.getenv("EBROOTLWP"),"lib64/liblwp_omp_publish.so")")
'''
moduleclass = 'system'
easyblock = 'SystemBundle'
name = 'LWP-settings'
version = 'LWP-procstat'
homepage = ''
description = 'Loads the LWP-procstat module for LWP'
toolchain = SYSTEM
modextravars = {
'atos_lwp_module_procstat': '1',
'LWP_MODULE_PROCSTAT_VERBOSE_LEVEL': '0',
}
moduleclass = 'system'
easyblock = 'SystemBundle'
name = 'LWP-settings'
version = 'LWP-sysinfo'
homepage = ''
description = 'Loads the LWP-sysinfo module for LWP'
toolchain = SYSTEM
modextravars = {
'atos_lwp_module_sysinfo': '1',
'LWP_MODULE_SYSINFO_VERBOSE_LEVEL': '0',
}
moduleclass = 'system'
easyblock = 'SystemBundle'
name = 'LWP-settings'
version = 'LWP-vmstat'
homepage = ''
description = 'Loads the LWP-vmstat module for LWP'
toolchain = SYSTEM
modextravars = {
'atos_lwp_module_vmstat': '1',
'LWP_MODULE_VMSTAT_VERBOSE_LEVEL': '0',
}
moduleclass = 'system'
easyblock = 'Rpm'
name = 'LWP'
version = '1.1.3'
local_lwp_omp_version = '1.0.4'
local_lwp_timestamp = '20210730182824'
homepage = ''
description = """Lightweight Profiler (LWP) is a lightweight tool for monitoring applications running on supercomputers.
It was designed to collect different kinds of useful data like CPU metrics, memory utilization, energy consumption, MPI
communication events, etc. It is used as a wrapper for an application; the user invokes LWP explicitly via the command
line. LWP is modular by its nature, so each type of data is collected by a specific component."""
toolchain = SYSTEM
sources = [
f'atos-lwp-{version}-Atos.{local_lwp_timestamp}.el8.x86_64.rpm',
f'atos-lwp-doc-{version}-Atos.{local_lwp_timestamp}.el8.noarch.rpm',
f'atos-lwp-omp-{local_lwp_omp_version}-Atos.20210730183139.el8.x86_64.rpm',
f'atos-lwp-procstat_module-{version}-Atos.{local_lwp_timestamp}.el8.x86_64.rpm',
f'atos-lwp-sysinfo_module-{version}-Atos.{local_lwp_timestamp}.el8.x86_64.rpm',
f'atos-lwp-vmstat_module-{version}-Atos.{local_lwp_timestamp}.el8.x86_64.rpm',
]
builddependencies = [
('rpmrebuild', '2.16', '', SYSTEM)
]
modextravars = {
'LWP_MODULES_DIR': f'%(installdir)s/lib64/atos-lwp',
'LWP': '%(installdir)s/lib64/liblwp_envmodule.so',
'LWP_VERBOSE_LEVEL': '0',
'LWP_DELAY': '5',
'MPI_AS_MODULE': '0',
}
modextrapaths = {
'LWP_LIBS': 'lib64/liblwp_envmodule.so'
}
local_path_to_fix = f'/opt/tools/profilers/atos-lwp'
postinstallcmds = [
# Move to bin/ lib64/ include/ libexec/ and share/ to the correct directory
f'mv %(installdir)s{local_path_to_fix}/* %(installdir)s',
# Clean the unneded directories
f'rm -Rf %(installdir)s/usr',
f'rm -Rf %(installdir)s/opt',
f'rm -Rf %(installdir)s/scripts',
f'rm -Rf %(installdir)s/rpm',
]
moduleclass = 'tools'
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment