Skip to content
Snippets Groups Projects
Commit 0b0916d6 authored by Damian Alvarez's avatar Damian Alvarez
Browse files

To hide things correctly and to use the correct MNS

parent 0f5a025b
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Custom_MNS/custom_hierarchical_mns.py → Custom_MNS/flexible_custom_hierarchical_mns.py
+ 94
− 32
View file @ 0b0916d6
##
# Copyright 2013-2016 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
# the Hercules foundation (http://www.herculesstichting.be/in_English)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# http://github.com/hpcugent/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
Implementation of an example hierarchical module naming scheme.
Implementation of a hierarchical module naming scheme, with added flexibility
@author: Kenneth Hoste (Ghent University)
@author: Markus Geimer (Forschungszentrum Juelich GmbH)
@author: Alan O'Cais (Forschungszentrum Juelich GmbH)
@author: Damian Alvarez (Forschungszentrum Juelich GmbH)
"""
import os
......@@ -40,6 +18,7 @@ from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compile
CORE = 'Core'
COMPILER = 'Compiler'
MPI = 'MPI'
MPI_SETTINGS = 'MPI_settings'
MODULECLASS_COMPILER = 'compiler'
MODULECLASS_MPI = 'mpi'
......@@ -54,7 +33,28 @@ COMP_NAME_VERSION_TEMPLATES = {
'xlc,xlf': ('xlcxlf', '%(xlc)s'),
}
class CustomHierarchicalMNS(HierarchicalMNS):
# Compiler relevant version numbers
comp_relevant_versions = {
'intel': 1,
'PGI': 1,
# The compilers load GCCcore/version. So GCC and GCCcore can't really be flexible, since GCCcore will always be loaded
# as a dependency with a full version, and GCC is nothing but a bundle around GCCcore + binutils
# 'GCC': 1,
# 'GCCcore': 1,
}
# MPI relevant version numbers
mpi_relevant_versions = {
'impi': 1,
'psmpi': 2,
'MVAPICH2': 2,
'OpenMPI': 2,
}
# MPIs with settings modules
mpi_with_settings = ['psmpi', 'impi', 'MVAPICH2']
class FlexibleCustomHierarchicalMNS(HierarchicalMNS):
"""Class implementing an example hierarchical module naming scheme."""
def is_short_modname_for(self, short_modname, name):
"""
......@@ -69,6 +69,8 @@ class CustomHierarchicalMNS(HierarchicalMNS):
modname_regex = re.compile('^%s/\S+$' % re.escape('ParaStationMPI'))
elif name == 'impi':
modname_regex = re.compile('^%s/\S+$' % re.escape('IntelMPI'))
elif name in ['-'.join([x, 'settings']) for x in mpi_with_settings]:
modname_regex = re.compile('^%s/\S+$' % re.escape('mpi-settings'))
else:
modname_regex = re.compile('^%s/\S+$' % re.escape(name))
res = bool(modname_regex.match(short_modname))
......@@ -78,6 +80,43 @@ class CustomHierarchicalMNS(HierarchicalMNS):
return res
def _find_relevant_compiler_info(self, comp_info):
comp_name, comp_ver = comp_info
# Strip the irrelevant bits of the version and append the suffix again
if comp_relevant_versions.has_key(comp_name):
suffix = '-'.join(comp_ver.split('-')[1:])
comp_ver = '.'.join(comp_ver.split('.')[:comp_relevant_versions[comp_name]])
if suffix:
comp_ver += '-%s' % suffix
return comp_name, comp_ver
def _find_relevant_mpi_info(self, mpi_info):
mpi_ver = self.det_full_version(mpi_info)
mpi_name = mpi_info['name']
# Find suffix, if any, to be appended. Try to be clever, since the suffix is embedded in the version
# and sometimes the version might include a string that looks like a suffix (ie: psmpi-5.4.0-1)
if mpi_relevant_versions.has_key(mpi_name):
# Find possible suffixes
possible_suffixes = mpi_ver.split('-')[1:]
# Match the '-1' that is a typical part of psmpi's version
if possible_suffixes:
if re.match('^\d$', possible_suffixes[0]):
suffix_index = 2
else:
suffix_index = 1
suffix = '-'.join(mpi_ver.split('-')[suffix_index:])
else:
suffix = ''
mpi_ver = '.'.join(mpi_ver.split('.')[:mpi_relevant_versions[mpi_name]])
if suffix:
mpi_ver += '-%s' % suffix
return mpi_name, mpi_ver
def det_module_subdir(self, ec):
"""
Determine module subdirectory, relative to the top of the module path.
......@@ -90,8 +129,12 @@ class CustomHierarchicalMNS(HierarchicalMNS):
if tc_comp_info is None:
# no compiler in toolchain, dummy toolchain => Core module
subdir = CORE
# except if the module is a MPI settings module
stripped_name = re.sub('-settings$', '', ec['name'])
if stripped_name in mpi_with_settings:
subdir = os.path.join(MPI_SETTINGS, stripped_name, ec['version'])
else:
tc_comp_name, tc_comp_ver = tc_comp_info
tc_comp_name, tc_comp_ver = self._find_relevant_compiler_info(tc_comp_info)
tc_mpi = det_toolchain_mpi(ec)
if tc_mpi is None:
# compiler-only toolchain => Compiler/<compiler_name>/<compiler_version> namespace
......@@ -101,12 +144,24 @@ class CustomHierarchicalMNS(HierarchicalMNS):
else:
subdir = os.path.join(COMPILER, tc_comp_name, tc_comp_ver)
else:
tc_mpi_name, tc_mpi_ver = self._find_relevant_mpi_info(tc_mpi)
# compiler-MPI toolchain => MPI/<comp_name>/<comp_version>/<MPI_name>/<MPI_version> namespace
tc_mpi_fullver = self.det_full_version(tc_mpi)
subdir = os.path.join(MPI, tc_comp_name, tc_comp_ver, tc_mpi['name'], tc_mpi_fullver)
subdir = os.path.join(MPI, tc_comp_name, tc_comp_ver, tc_mpi_name, tc_mpi_ver)
return subdir
def det_short_module_name(self, ec):
"""
Determine short module name, i.e. the name under which modules will be exposed to users.
Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5
UCX-UD (for MPI settings)
"""
stripped_name = re.sub('-settings$', '', ec['name'])
if stripped_name in mpi_with_settings and '-settings' in ec['name']:
return os.path.join('mpi-settings', ec['versionsuffix'])
else:
return super(FlexibleCustomHierarchicalMNS, self).det_short_module_name(ec)
def det_modpath_extensions(self, ec):
"""
Determine module path extensions, if any.
......@@ -146,20 +201,27 @@ class CustomHierarchicalMNS(HierarchicalMNS):
# This means icc/ifort are not of the moduleclass compiler but iccifort is
if ec['name'] == 'iccifort':
comp_name_ver = ['intel', self.det_full_version(ec)]
# Exclude extending the path for icc/ifort, the iccifort special case is handled above
if ec['name'] not in ['icc', 'ifort']:
# Overwrite version if necessary
comp_name_ver = self._find_relevant_compiler_info(comp_name_ver)
paths.append(os.path.join(COMPILER, *comp_name_ver))
# Always extend to capture the MPI implementations too (which are in a separate directory)
if ec['name'] not in [GCCCORE]:
paths.append(os.path.join(COMPILER, MODULECLASS_MPI, *comp_name_ver))
elif modclass == MODULECLASS_MPI:
if tc_comp_info is None:
raise EasyBuildError("No compiler available in toolchain %s used to install MPI library %s v%s, "
"which is required by the active module naming scheme.",
ec['toolchain'], ec['name'], ec['version'])
else:
tc_comp_name, tc_comp_ver = tc_comp_info
fullver = self.det_full_version(ec)
paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver))
tc_comp_name, tc_comp_ver = self._find_relevant_compiler_info(tc_comp_info)
mpi_name, mpi_ver = self._find_relevant_mpi_info(ec)
paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, mpi_name, mpi_ver))
if ec['name'] in mpi_with_settings:
paths.append(os.path.join(MPI_SETTINGS, mpi_name, mpi_ver))
return paths
Golden_Repo/hidden_deps.txt 0 → 100644
+ 221
− 0
View file @ 0b0916d6
adwaita-icon-theme
ant
ANTLR
APR
APR-util
assimp
ATK
AT-SPI2-ATK
AT-SPI2-core
Autoconf
Automake
babl
binutils
Bison
byacc
bzip2
cairo
cling
configurable-http-proxy
Coreutils
CUSP
damageproto
DB
DBus
dbus-glib
DocBook-XML
Dyninst
ETSF_IO
eudev
Exiv2
expat
FFmpeg
fixesproto
FLTK
fontconfig
fontsproto
FoX
freeglut
freetype
FTGL
g2clib
g2lib
gc
GCCcore
GDAL
Gdk-Pixbuf
GEGL
gettext
gexiv2
gflags
Ghostscript
GL2PS
GLEW
GLib
GLM
glog
GLPK
glproto
GObject-Introspection
googletest
GPC
gperf
GraphicsMagick
gsettings-desktop-schemas
GTI
GTK+
GtkSourceView
GTS
guile
gzip
HarfBuzz
icc
ICU
ifort
inputproto
intltool
itstool
JasPer
jhbuild
JSON-C
JSON-GLib
JUnit
kbproto
LevelDB
libcerf
libcroco
libctl
libdap
libdrm
libdwarf
libelf
libepoxy
libevent
libffi
libfontenc
libgd
libgeotiff
libglade
libGLU
libICE
libidn
Libint
libjpeg-turbo
libmatheval
libmypaint
libpciaccess
libpng
libpthread-stubs
libreadline
librsvg
libSM
libsndfile
libspatialindex
LibTIFF
libtool
libunistring
libunwind
LibUUID
libX11
libXau
libXaw
libxcb
libXcursor
libXdamage
libXdmcp
libXext
libXfixes
libXfont
libXft
libXi
libXinerama
libxkbcommon
libxml2
libXmu
libXp
libXpm
libXrandr
libXrender
libxslt
libXt
libXtst
libyaml
libyuv
LittleCMS
LMDB
LZO
M4
makedepend
MATIO
motif
msgpack-c
NASM
ncurses
nettle
NLopt
nodejs
nvenc_sdk
nvidia
OPARI2
OTF2
Pango
patchelf
PCRE
PDT
pixman
pkgconfig
pkg-config
Pmw
PnMPI
popt
printproto
PROJ
protobuf
pscom
pybind11
PyCairo
PyGObject
Python-Xpra
Qhull
qrupdate
Qt
Qt5
randrproto
recordproto
renderproto
SCons
scrollkeeper
Serf
SIP
S-Lang
snappy
SQLite
SWIG
Szip
texinfo
Tk
UDUNITS
util-linux
vpx
wxPropertyGrid
wxWidgets
x264
x265
xbitmaps
xcb-proto
xcb-util
xcb-util-image
xcb-util-keysyms
xcb-util-renderutil
xcb-util-wm
xextproto
xineramaproto
XKeyboardConfig
XML-Parser
xorg-macros
xprop
xproto
xtrans
XZ
Yasm
zlib
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment