diff --git a/Custom_MNS/custom_hierarchical_mns.py b/Custom_MNS/flexible_custom_hierarchical_mns.py
similarity index 61%
rename from Custom_MNS/custom_hierarchical_mns.py
rename to Custom_MNS/flexible_custom_hierarchical_mns.py
index 3c5ae5bd6bf8e2f70c88f99dfb8801fb62454843..dd4c0c97503f49877b60f1db8047c11f6cb10814 100644
--- a/Custom_MNS/custom_hierarchical_mns.py
+++ b/Custom_MNS/flexible_custom_hierarchical_mns.py
@@ -1,32 +1,10 @@
-##
-# Copyright 2013-2016 Ghent University
-#
-# This file is part of EasyBuild,
-# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
-# with support of Ghent University (http://ugent.be/hpc),
-# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
-# the Hercules foundation (http://www.herculesstichting.be/in_English)
-# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
-#
-# http://github.com/hpcugent/easybuild
-#
-# EasyBuild is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation v2.
-#
-# EasyBuild is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
-##
"""
-Implementation of an example hierarchical module naming scheme.
+Implementation of a hierarchical module naming scheme, with added flexibility
@author: Kenneth Hoste (Ghent University)
@author: Markus Geimer (Forschungszentrum Juelich GmbH)
+@author: Alan O'Cais (Forschungszentrum Juelich GmbH)
+@author: Damian Alvarez (Forschungszentrum Juelich GmbH)
"""
import os
@@ -40,6 +18,7 @@ from easybuild.tools.module_naming_scheme.toolchain import det_toolchain_compile
CORE = 'Core'
COMPILER = 'Compiler'
MPI = 'MPI'
+MPI_SETTINGS = 'MPI_settings'
MODULECLASS_COMPILER = 'compiler'
MODULECLASS_MPI = 'mpi'
@@ -54,7 +33,28 @@ COMP_NAME_VERSION_TEMPLATES = {
'xlc,xlf': ('xlcxlf', '%(xlc)s'),
}
-class CustomHierarchicalMNS(HierarchicalMNS):
+# Compiler relevant version numbers
+comp_relevant_versions = {
+ 'intel': 1,
+ 'PGI': 1,
+# The compilers load GCCcore/version. So GCC and GCCcore can't really be flexible, since GCCcore will always be loaded
+# as a dependency with a full version, and GCC is nothing but a bundle around GCCcore + binutils
+# 'GCC': 1,
+# 'GCCcore': 1,
+}
+
+# MPI relevant version numbers
+mpi_relevant_versions = {
+ 'impi': 1,
+ 'psmpi': 2,
+ 'MVAPICH2': 2,
+ 'OpenMPI': 2,
+}
+
+# MPIs with settings modules
+mpi_with_settings = ['psmpi', 'impi', 'MVAPICH2']
+
+class FlexibleCustomHierarchicalMNS(HierarchicalMNS):
"""Class implementing an example hierarchical module naming scheme."""
def is_short_modname_for(self, short_modname, name):
"""
@@ -69,6 +69,8 @@ class CustomHierarchicalMNS(HierarchicalMNS):
modname_regex = re.compile('^%s/\S+$' % re.escape('ParaStationMPI'))
elif name == 'impi':
modname_regex = re.compile('^%s/\S+$' % re.escape('IntelMPI'))
+ elif name in ['-'.join([x, 'settings']) for x in mpi_with_settings]:
+ modname_regex = re.compile('^%s/\S+$' % re.escape('mpi-settings'))
else:
modname_regex = re.compile('^%s/\S+$' % re.escape(name))
res = bool(modname_regex.match(short_modname))
@@ -78,6 +80,43 @@ class CustomHierarchicalMNS(HierarchicalMNS):
return res
+ def _find_relevant_compiler_info(self, comp_info):
+ comp_name, comp_ver = comp_info
+
+ # Strip the irrelevant bits of the version and append the suffix again
+ if comp_relevant_versions.has_key(comp_name):
+ suffix = '-'.join(comp_ver.split('-')[1:])
+ comp_ver = '.'.join(comp_ver.split('.')[:comp_relevant_versions[comp_name]])
+ if suffix:
+ comp_ver += '-%s' % suffix
+
+ return comp_name, comp_ver
+
+ def _find_relevant_mpi_info(self, mpi_info):
+ mpi_ver = self.det_full_version(mpi_info)
+ mpi_name = mpi_info['name']
+
+ # Find suffix, if any, to be appended. Try to be clever, since the suffix is embedded in the version
+ # and sometimes the version might include a string that looks like a suffix (ie: psmpi-5.4.0-1)
+ if mpi_relevant_versions.has_key(mpi_name):
+ # Find possible suffixes
+ possible_suffixes = mpi_ver.split('-')[1:]
+ # Match the '-1' that is a typical part of psmpi's version
+ if possible_suffixes:
+ if re.match('^\d$', possible_suffixes[0]):
+ suffix_index = 2
+ else:
+ suffix_index = 1
+ suffix = '-'.join(mpi_ver.split('-')[suffix_index:])
+ else:
+ suffix = ''
+
+ mpi_ver = '.'.join(mpi_ver.split('.')[:mpi_relevant_versions[mpi_name]])
+ if suffix:
+ mpi_ver += '-%s' % suffix
+
+ return mpi_name, mpi_ver
+
def det_module_subdir(self, ec):
"""
Determine module subdirectory, relative to the top of the module path.
@@ -90,8 +129,12 @@ class CustomHierarchicalMNS(HierarchicalMNS):
if tc_comp_info is None:
# no compiler in toolchain, dummy toolchain => Core module
subdir = CORE
+ # except if the module is a MPI settings module
+ stripped_name = re.sub('-settings$', '', ec['name'])
+ if stripped_name in mpi_with_settings:
+ subdir = os.path.join(MPI_SETTINGS, stripped_name, ec['version'])
else:
- tc_comp_name, tc_comp_ver = tc_comp_info
+ tc_comp_name, tc_comp_ver = self._find_relevant_compiler_info(tc_comp_info)
tc_mpi = det_toolchain_mpi(ec)
if tc_mpi is None:
# compiler-only toolchain => Compiler/<compiler_name>/<compiler_version> namespace
@@ -101,12 +144,24 @@ class CustomHierarchicalMNS(HierarchicalMNS):
else:
subdir = os.path.join(COMPILER, tc_comp_name, tc_comp_ver)
else:
+ tc_mpi_name, tc_mpi_ver = self._find_relevant_mpi_info(tc_mpi)
# compiler-MPI toolchain => MPI/<comp_name>/<comp_version>/<MPI_name>/<MPI_version> namespace
- tc_mpi_fullver = self.det_full_version(tc_mpi)
- subdir = os.path.join(MPI, tc_comp_name, tc_comp_ver, tc_mpi['name'], tc_mpi_fullver)
+ subdir = os.path.join(MPI, tc_comp_name, tc_comp_ver, tc_mpi_name, tc_mpi_ver)
return subdir
+ def det_short_module_name(self, ec):
+ """
+ Determine short module name, i.e. the name under which modules will be exposed to users.
+ Examples: GCC/4.8.3, OpenMPI/1.6.5, OpenBLAS/0.2.9, HPL/2.1, Python/2.7.5
+ UCX-UD (for MPI settings)
+ """
+ stripped_name = re.sub('-settings$', '', ec['name'])
+ if stripped_name in mpi_with_settings and '-settings' in ec['name']:
+ return os.path.join('mpi-settings', ec['versionsuffix'])
+ else:
+ return super(FlexibleCustomHierarchicalMNS, self).det_short_module_name(ec)
+
def det_modpath_extensions(self, ec):
"""
Determine module path extensions, if any.
@@ -146,20 +201,27 @@ class CustomHierarchicalMNS(HierarchicalMNS):
# This means icc/ifort are not of the moduleclass compiler but iccifort is
if ec['name'] == 'iccifort':
comp_name_ver = ['intel', self.det_full_version(ec)]
+
# Exclude extending the path for icc/ifort, the iccifort special case is handled above
if ec['name'] not in ['icc', 'ifort']:
+ # Overwrite version if necessary
+ comp_name_ver = self._find_relevant_compiler_info(comp_name_ver)
paths.append(os.path.join(COMPILER, *comp_name_ver))
# Always extend to capture the MPI implementations too (which are in a separate directory)
if ec['name'] not in [GCCCORE]:
paths.append(os.path.join(COMPILER, MODULECLASS_MPI, *comp_name_ver))
+
elif modclass == MODULECLASS_MPI:
if tc_comp_info is None:
raise EasyBuildError("No compiler available in toolchain %s used to install MPI library %s v%s, "
"which is required by the active module naming scheme.",
ec['toolchain'], ec['name'], ec['version'])
else:
- tc_comp_name, tc_comp_ver = tc_comp_info
- fullver = self.det_full_version(ec)
- paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, ec['name'], fullver))
+ tc_comp_name, tc_comp_ver = self._find_relevant_compiler_info(tc_comp_info)
+ mpi_name, mpi_ver = self._find_relevant_mpi_info(ec)
+ paths.append(os.path.join(MPI, tc_comp_name, tc_comp_ver, mpi_name, mpi_ver))
+
+ if ec['name'] in mpi_with_settings:
+ paths.append(os.path.join(MPI_SETTINGS, mpi_name, mpi_ver))
return paths
diff --git a/Golden_Repo/hidden_deps.txt b/Golden_Repo/hidden_deps.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9825e2dd98dba248f89648817883be8e5eda3820
--- /dev/null
+++ b/Golden_Repo/hidden_deps.txt
@@ -0,0 +1,221 @@
+adwaita-icon-theme
+ant
+ANTLR
+APR
+APR-util
+assimp
+ATK
+AT-SPI2-ATK
+AT-SPI2-core
+Autoconf
+Automake
+babl
+binutils
+Bison
+byacc
+bzip2
+cairo
+cling
+configurable-http-proxy
+Coreutils
+CUSP
+damageproto
+DB
+DBus
+dbus-glib
+DocBook-XML
+Dyninst
+ETSF_IO
+eudev
+Exiv2
+expat
+FFmpeg
+fixesproto
+FLTK
+fontconfig
+fontsproto
+FoX
+freeglut
+freetype
+FTGL
+g2clib
+g2lib
+gc
+GCCcore
+GDAL
+Gdk-Pixbuf
+GEGL
+gettext
+gexiv2
+gflags
+Ghostscript
+GL2PS
+GLEW
+GLib
+GLM
+glog
+GLPK
+glproto
+GObject-Introspection
+googletest
+GPC
+gperf
+GraphicsMagick
+gsettings-desktop-schemas
+GTI
+GTK+
+GtkSourceView
+GTS
+guile
+gzip
+HarfBuzz
+icc
+ICU
+ifort
+inputproto
+intltool
+itstool
+JasPer
+jhbuild
+JSON-C
+JSON-GLib
+JUnit
+kbproto
+LevelDB
+libcerf
+libcroco
+libctl
+libdap
+libdrm
+libdwarf
+libelf
+libepoxy
+libevent
+libffi
+libfontenc
+libgd
+libgeotiff
+libglade
+libGLU
+libICE
+libidn
+Libint
+libjpeg-turbo
+libmatheval
+libmypaint
+libpciaccess
+libpng
+libpthread-stubs
+libreadline
+librsvg
+libSM
+libsndfile
+libspatialindex
+LibTIFF
+libtool
+libunistring
+libunwind
+LibUUID
+libX11
+libXau
+libXaw
+libxcb
+libXcursor
+libXdamage
+libXdmcp
+libXext
+libXfixes
+libXfont
+libXft
+libXi
+libXinerama
+libxkbcommon
+libxml2
+libXmu
+libXp
+libXpm
+libXrandr
+libXrender
+libxslt
+libXt
+libXtst
+libyaml
+libyuv
+LittleCMS
+LMDB
+LZO
+M4
+makedepend
+MATIO
+motif
+msgpack-c
+NASM
+ncurses
+nettle
+NLopt
+nodejs
+nvenc_sdk
+nvidia
+OPARI2
+OTF2
+Pango
+patchelf
+PCRE
+PDT
+pixman
+pkgconfig
+pkg-config
+Pmw
+PnMPI
+popt
+printproto
+PROJ
+protobuf
+pscom
+pybind11
+PyCairo
+PyGObject
+Python-Xpra
+Qhull
+qrupdate
+Qt
+Qt5
+randrproto
+recordproto
+renderproto
+SCons
+scrollkeeper
+Serf
+SIP
+S-Lang
+snappy
+SQLite
+SWIG
+Szip
+texinfo
+Tk
+UDUNITS
+util-linux
+vpx
+wxPropertyGrid
+wxWidgets
+x264
+x265
+xbitmaps
+xcb-proto
+xcb-util
+xcb-util-image
+xcb-util-keysyms
+xcb-util-renderutil
+xcb-util-wm
+xextproto
+xineramaproto
+XKeyboardConfig
+XML-Parser
+xorg-macros
+xprop
+xproto
+xtrans
+XZ
+Yasm
+zlib