From f112c6edc5fe8b01cc8f90c0ec9ab169d89da01b Mon Sep 17 00:00:00 2001
From: Damian Alvarez <swmanage@jwlogin24.juwels>
Date: Wed, 19 May 2021 18:35:15 +0200
Subject: [PATCH] Update of the overlay used for jurecabooster
---
.gitignore | 1 +
.../jurecabooster_overlay/g/GCC/GCC-9.3.0.eb | 31 +++++++++
.../g/GCCcore/GCCcore-9.3.0.eb | 67 +++++++++++++++++++
.../h/hwloc/hwloc-2.2.0-GCCcore-9.3.0.eb | 49 ++++++++++++++
.../libfabric-1.12.1-GCCcore-9.3.0.eb | 41 ++++++++++++
.../OpenMPI-settings-4.1-PSM.eb | 43 ++++++++++++
.../o/OpenMPI/OpenMPI-4.1.0rc1-GCC-9.3.0.eb | 63 +++++++++++++++++
...-4.1.0rc1-iccifort-2020.2.254-GCC-9.3.0.eb | 63 +++++++++++++++++
.../o/OpenMPI/psm2_pmix.patch | 47 +++++++++++++
9 files changed, 405 insertions(+)
create mode 100644 Overlays/jurecabooster_overlay/g/GCC/GCC-9.3.0.eb
create mode 100644 Overlays/jurecabooster_overlay/g/GCCcore/GCCcore-9.3.0.eb
create mode 100644 Overlays/jurecabooster_overlay/h/hwloc/hwloc-2.2.0-GCCcore-9.3.0.eb
create mode 100644 Overlays/jurecabooster_overlay/l/libfabric/libfabric-1.12.1-GCCcore-9.3.0.eb
create mode 100644 Overlays/jurecabooster_overlay/o/OpenMPI-settings/OpenMPI-settings-4.1-PSM.eb
create mode 100644 Overlays/jurecabooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-GCC-9.3.0.eb
create mode 100644 Overlays/jurecabooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-iccifort-2020.2.254-GCC-9.3.0.eb
create mode 100644 Overlays/jurecabooster_overlay/o/OpenMPI/psm2_pmix.patch
diff --git a/.gitignore b/.gitignore
index 338638cc6..e9af798e8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
# Ignore any pyc files
*.pyc
+*.pyo
.DS_Store
._.DS_Store
diff --git a/Overlays/jurecabooster_overlay/g/GCC/GCC-9.3.0.eb b/Overlays/jurecabooster_overlay/g/GCC/GCC-9.3.0.eb
new file mode 100644
index 000000000..74b95b877
--- /dev/null
+++ b/Overlays/jurecabooster_overlay/g/GCC/GCC-9.3.0.eb
@@ -0,0 +1,31 @@
+easyblock = 'Bundle'
+
+name = 'GCC'
+version = '9.3.0'
+
+homepage = 'http://gcc.gnu.org/'
+description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada,
+as well as libraries for these languages (libstdc++, libgcj,...).
+"""
+
+site_contacts = 'Damian Alvarez <d.alvarez@fz-juelich.de>'
+
+toolchain = SYSTEM
+
+dependencies = [
+ ('GCCcore', version),
+ # binutils built on top of GCCcore, which was built on top of (dummy-built) binutils
+ ('binutils', '2.34', '', ('GCCcore', version)),
+]
+
+altroot = 'GCCcore'
+altversion = 'GCCcore'
+
+# We use a HMNS, so let's enforce a unique compiler
+modluafooter = 'family("compiler")'
+
+# Always do a recursive unload on compilers
+recursive_module_unload = True
+
+# this bundle serves as a compiler toolchain, so it should be marked as compiler (important for HMNS)
+moduleclass = 'compiler'
diff --git a/Overlays/jurecabooster_overlay/g/GCCcore/GCCcore-9.3.0.eb b/Overlays/jurecabooster_overlay/g/GCCcore/GCCcore-9.3.0.eb
new file mode 100644
index 000000000..1ec4b5d7f
--- /dev/null
+++ b/Overlays/jurecabooster_overlay/g/GCCcore/GCCcore-9.3.0.eb
@@ -0,0 +1,67 @@
+easyblock = 'EB_GCC'
+
+name = 'GCCcore'
+version = '9.3.0'
+local_newlib_version = '3.3.0'
+
+homepage = 'https://gcc.gnu.org/'
+description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada,
+as well as libraries for these languages (libstdc++, libgcj,...).
+"""
+
+site_contacts = 'Damian Alvarez <d.alvarez@fz-juelich.de>'
+
+toolchain = SYSTEM
+
+source_urls = [
+ 'https://ftpmirror.gnu.org/gnu/gcc/gcc-%(version)s', # GCC auto-resolving HTTP mirror
+ 'https://ftpmirror.gnu.org/gnu/gmp', # idem for GMP
+ 'https://ftpmirror.gnu.org/gnu/mpfr', # idem for MPFR
+ 'https://ftpmirror.gnu.org/gnu/mpc', # idem for MPC
+ 'ftp://gcc.gnu.org/pub/gcc/infrastructure/', # GCC dependencies
+ 'http://gcc.cybermirror.org/infrastructure/', # HTTP mirror for GCC dependencies
+ 'http://isl.gforge.inria.fr/', # original HTTP source for ISL
+ 'https://sourceware.org/pub/newlib/', # for newlib
+ 'https://github.com/MentorEmbedded/nvptx-tools/archive', # for nvptx-tools
+]
+sources = [
+ 'gcc-%(version)s.tar.gz',
+ 'gmp-6.2.0.tar.bz2',
+ 'mpfr-4.0.2.tar.bz2',
+ 'mpc-1.1.0.tar.gz',
+ 'isl-0.22.1.tar.bz2',
+ 'newlib-3.3.0.tar.gz',
+]
+patches = [
+ 'GCCcore-6.2.0-fix-find-isl.patch',
+ 'GCCcore-8.3.0_fix-xsmin-ppc.patch',
+ 'GCCcore-%(version)s_gmp-c99.patch',
+ 'GCCcore-%(version)s_vect_broadcasts_masmintel.patch',
+]
+checksums = [
+ '5258a9b6afe9463c2e56b9e8355b1a4bee125ca828b8078f910303bc2ef91fa6', # gcc-9.3.0.tar.gz
+ 'f51c99cb114deb21a60075ffb494c1a210eb9d7cb729ed042ddb7de9534451ea', # gmp-6.2.0.tar.bz2
+ 'c05e3f02d09e0e9019384cdd58e0f19c64e6db1fd6f5ecf77b4b1c61ca253acc', # mpfr-4.0.2.tar.bz2
+ '6985c538143c1208dcb1ac42cedad6ff52e267b47e5f970183a3e75125b43c2e', # mpc-1.1.0.tar.gz
+ '1a668ef92eb181a7c021e8531a3ca89fd71aa1b3744db56f68365ab0a224c5cd', # isl-0.22.1.tar.bz2
+ '58dd9e3eaedf519360d92d84205c3deef0b3fc286685d1c562e245914ef72c66', # newlib-3.3.0.tar.gz
+ '5ad909606d17d851c6ad629b4fddb6c1621844218b8d139fed18c502a7696c68', # GCCcore-6.2.0-fix-find-isl.patch
+ 'bea1bce8f50ea6d51b038309eb61dec00a8681fb653d211c539be80f184609a3', # GCCcore-8.3.0_fix-xsmin-ppc.patch
+ '0e135e1cc7cec701beea9d7d17a61bab34cfd496b4b555930016b98db99f922e', # GCCcore-9.3.0_gmp-c99.patch
+ 'a32ac9c7d999a8b91bf93dba6a9d81b6ff58b3c89c425ff76090cbc90076685c', # GCCcore-9.3.0_vect_broadcasts_masmintel.patch
+]
+
+builddependencies = [
+ ('M4', '1.4.18'),
+ ('binutils', '2.34'),
+]
+
+languages = ['c', 'c++', 'fortran']
+
+withisl = True
+withnvptx = False
+
+# Make sure we replace the system cc with gcc with an alias
+modaliases = {'cc': 'gcc'}
+
+moduleclass = 'compiler'
diff --git a/Overlays/jurecabooster_overlay/h/hwloc/hwloc-2.2.0-GCCcore-9.3.0.eb b/Overlays/jurecabooster_overlay/h/hwloc/hwloc-2.2.0-GCCcore-9.3.0.eb
new file mode 100644
index 000000000..f4a477b04
--- /dev/null
+++ b/Overlays/jurecabooster_overlay/h/hwloc/hwloc-2.2.0-GCCcore-9.3.0.eb
@@ -0,0 +1,49 @@
+easyblock = 'ConfigureMake'
+
+name = 'hwloc'
+version = '2.2.0'
+
+homepage = 'http://www.open-mpi.org/projects/hwloc/'
+
+description = """
+ The Portable Hardware Locality (hwloc) software package provides a portable
+ abstraction (across OS, versions, architectures, ...) of the hierarchical
+ topology of modern architectures, including NUMA memory nodes, sockets, shared
+ caches, cores and simultaneous multithreading. It also gathers various system
+ attributes such as cache and memory information as well as the locality of I/O
+ devices such as network interfaces, InfiniBand HCAs or GPUs. It primarily
+ aims at helping applications with gathering information about modern computing
+ hardware so as to exploit it accordingly and efficiently.
+"""
+
+site_contacts = 'sc@fz-juelich.de'
+
+toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
+
+source_urls = ['https://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/']
+sources = [SOURCE_TAR_GZ]
+
+builddependencies = [
+ ('binutils', '2.34'),
+]
+
+dependencies = [
+ ('numactl', '2.0.13', '', SYSTEM),
+ ('libxml2', '2.9.10'),
+ ('libpciaccess', '0.16'),
+]
+
+configopts = "--enable-libnuma=$EBROOTNUMACTL --enable-opencl"
+configopts += "--disable-cairo --disable-gl --disable-libudev "
+
+sanity_check_paths = {
+ 'files': ['bin/lstopo', 'include/hwloc/linux.h',
+ 'lib/libhwloc.%s' % SHLIB_EXT],
+ 'dirs': ['share/man/man3'],
+}
+sanity_check_commands = ['lstopo']
+
+modluafooter = '''
+add_property("arch","gpu")
+'''
+moduleclass = 'system'
diff --git a/Overlays/jurecabooster_overlay/l/libfabric/libfabric-1.12.1-GCCcore-9.3.0.eb b/Overlays/jurecabooster_overlay/l/libfabric/libfabric-1.12.1-GCCcore-9.3.0.eb
new file mode 100644
index 000000000..137a9a69c
--- /dev/null
+++ b/Overlays/jurecabooster_overlay/l/libfabric/libfabric-1.12.1-GCCcore-9.3.0.eb
@@ -0,0 +1,41 @@
+easyblock = 'ConfigureMake'
+
+name = 'libfabric'
+version = '1.12.1'
+
+homepage = 'https://ofiwg.github.io/libfabric/'
+description = """
+Libfabric is a core component of OFI. It is the library that defines and exports
+the user-space API of OFI, and is typically the only software that applications
+deal with directly. It works in conjunction with provider libraries, which are
+often integrated directly into libfabric.
+"""
+
+site_contacts = 'd.alvarez@fz-juelich.de'
+
+toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
+toolchainopts = {'pic': True}
+
+github_account = 'ofiwg'
+source_urls = ['https://github.com/ofiwg/%(name)s/releases/download/v%(version)s']
+sources = [SOURCE_TAR_BZ2]
+
+builddependencies = [
+ ('binutils', '2.34'),
+ ('pkg-config', '0.29.2'),
+]
+
+osdependencies = [OS_PKG_IBVERBS_DEV]
+
+# Disable deprecated "sockets" provider
+configopts = "--disable-sockets"
+
+sanity_check_paths = {
+ 'files': ['bin/fi_info', 'bin/fi_pingpong', 'bin/fi_strerror'] +
+ ['lib/libfabric.%s' % x for x in ['a', SHLIB_EXT]],
+ 'dirs': ['include/rdma', 'lib/pkgconfig', 'share']
+}
+
+sanity_check_commands = ['fi_info']
+
+moduleclass = 'lib'
diff --git a/Overlays/jurecabooster_overlay/o/OpenMPI-settings/OpenMPI-settings-4.1-PSM.eb b/Overlays/jurecabooster_overlay/o/OpenMPI-settings/OpenMPI-settings-4.1-PSM.eb
new file mode 100644
index 000000000..42d8d884c
--- /dev/null
+++ b/Overlays/jurecabooster_overlay/o/OpenMPI-settings/OpenMPI-settings-4.1-PSM.eb
@@ -0,0 +1,43 @@
+easyblock = 'SystemBundle'
+
+name = 'OpenMPI-settings'
+version = '4.1'
+versionsuffix = 'PSM'
+
+homepage = ''
+description = """This is a module to load the default OpenMPI configuration"""
+
+site_contacts = 'd.alvarez@fz-juelich.de'
+
+toolchain = SYSTEM
+
+source_urls = []
+
+sources = []
+modextravars = {
+ 'SLURM_MPI_TYPE': 'pspmix',
+ 'OMPI_MCA_mca_base_component_show_load_errors': '1',
+ 'OMPI_MCA_mpi_param_check': '1',
+ 'OMPI_MCA_mpi_show_handle_leaks': '1',
+ 'OMPI_MCA_mpi_warn_on_fork': '1',
+ # Disable uct for the time being due to:
+ # https://github.com/openucx/ucx/wiki/OpenMPI-and-OpenSHMEM-installation-with-UCX#running-open-mpi-with-ucx
+ # Also openib, since it is deprecated and should be substituted by the UCX support in the pml
+ 'OMPI_MCA_btl': 'ofi',
+ 'OMPI_MCA_mtl': 'ofi',
+# 'OMPI_MCA_btl_openib_allow_ib': '1',
+# 'OMPI_MCA_bml_r2_show_unreach_errors': '0',
+# 'OMPI_MCA_coll': '^ml',
+# 'OMPI_MCA_coll_hcoll_enable': '1',
+# 'OMPI_MCA_coll_hcoll_np': '0',
+# 'OMPI_MCA_pml': 'ucx',
+# 'OMPI_MCA_osc': '^rdma',
+# 'OMPI_MCA_opal_abort_print_stack': '1',
+# 'OMPI_MCA_opal_set_max_sys_limits': '1',
+# 'OMPI_MCA_opal_event_include': 'epoll',
+# 'OMPI_MCA_btl_openib_warn_default_gid_prefix': '0',
+ # OMPIO does not seem to work reliably on our system
+ 'OMPI_MCA_io': 'romio321',
+}
+
+moduleclass = 'system'
diff --git a/Overlays/jurecabooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-GCC-9.3.0.eb b/Overlays/jurecabooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-GCC-9.3.0.eb
new file mode 100644
index 000000000..2107e7804
--- /dev/null
+++ b/Overlays/jurecabooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-GCC-9.3.0.eb
@@ -0,0 +1,63 @@
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.0rc1'
+
+homepage = 'http://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+site_contacts = 'sc@fz-juelich.de'
+
+toolchain = {'name': 'GCC', 'version': '9.3.0'}
+
+source_urls = ['http://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_GZ]
+
+patches = ['psm2_pmix.patch']
+
+osdependencies = [
+ # needed for --with-pmix
+ ('pmix-devel'),
+]
+
+builddependencies = [
+ ('Autotools', '20200321'),
+]
+
+dependencies = [
+ ('zlib', '1.2.11'),
+ ('hwloc', '2.2.0'),
+ ('libevent', '2.1.12'),
+ ('libfabric', '1.12.1'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC ' # hwloc support
+configopts += '--with-psm2 '
+configopts += '--with-ofi=$EBROOTLIBFABRIC '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--disable-oshmem '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+ 'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+ ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+ ["include/%s.h" % x for x in ["mpi-ext", "mpif-config", "mpif", "mpi", "mpi_portable_platform"]],
+ 'dirs': [],
+}
+
+# Add a family for our naming scheme
+modluafooter = '''
+family("mpi")
+add_property("arch","gpu")
+if not ( isloaded("mpi-settings") ) then
+ load("mpi-settings")
+end
+'''
+
+moduleclass = 'mpi'
diff --git a/Overlays/jurecabooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-iccifort-2020.2.254-GCC-9.3.0.eb b/Overlays/jurecabooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-iccifort-2020.2.254-GCC-9.3.0.eb
new file mode 100644
index 000000000..2c2912833
--- /dev/null
+++ b/Overlays/jurecabooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-iccifort-2020.2.254-GCC-9.3.0.eb
@@ -0,0 +1,63 @@
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.0rc1'
+
+homepage = 'http://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+site_contacts = 'sc@fz-juelich.de'
+
+toolchain = {'name': 'iccifort', 'version': '2020.2.254-GCC-9.3.0'}
+
+source_urls = ['http://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_GZ]
+
+patches = ['psm2_pmix.patch']
+
+osdependencies = [
+ # needed for --with-pmix
+ ('pmix-devel'),
+]
+
+builddependencies = [
+ ('Autotools', '20200321'),
+]
+
+dependencies = [
+ ('zlib', '1.2.11'),
+ ('hwloc', '2.2.0'),
+ ('libevent', '2.1.12'),
+ ('libfabric', '1.12.1'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC ' # hwloc support
+configopts += '--with-psm2 '
+configopts += '--with-ofi=$EBROOTLIBFABRIC '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--disable-oshmem '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+ 'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+ ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+ ["include/%s.h" % x for x in ["mpi-ext", "mpif-config", "mpif", "mpi", "mpi_portable_platform"]],
+ 'dirs': [],
+}
+
+# Add a family for our naming scheme
+modluafooter = '''
+family("mpi")
+add_property("arch","gpu")
+if not ( isloaded("mpi-settings") ) then
+ load("mpi-settings")
+end
+'''
+
+moduleclass = 'mpi'
diff --git a/Overlays/jurecabooster_overlay/o/OpenMPI/psm2_pmix.patch b/Overlays/jurecabooster_overlay/o/OpenMPI/psm2_pmix.patch
new file mode 100644
index 000000000..9689fbddb
--- /dev/null
+++ b/Overlays/jurecabooster_overlay/o/OpenMPI/psm2_pmix.patch
@@ -0,0 +1,47 @@
+diff -ruN openmpi-4.1.0rc1.orig/ompi/mca/mtl/psm2/mtl_psm2.c openmpi-4.1.0rc1.patched/ompi/mca/mtl/psm2/mtl_psm2.c
+--- openmpi-4.1.0rc1.orig/ompi/mca/mtl/psm2/mtl_psm2.c 2020-07-06 23:56:33.000000000 +0200
++++ openmpi-4.1.0rc1.patched/ompi/mca/mtl/psm2/mtl_psm2.c 2021-05-11 12:05:35.399831599 +0200
+@@ -26,6 +26,7 @@
+ #include "ompi_config.h"
+
+ #include "opal/mca/pmix/pmix.h"
++#include "opal/mca/pmix/pmix3x/pmix/include/pmix_common.h"
+ #include "ompi/mca/mtl/mtl.h"
+ #include "ompi/mca/mtl/base/mtl_base_datatype.h"
+ #include "opal/util/show_help.h"
+@@ -102,6 +103,21 @@
+ int rc;
+
+ generated_key = getenv(OPAL_MCA_PREFIX"orte_precondition_transports");
++ opal_process_name_t pname;
++
++ generated_key = NULL;
++ pname.jobid = atoi(getenv("SLURM_JOBID"));
++ pname.vpid = OPAL_VPID_WILDCARD;
++ OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_CREDENTIAL, &pname,
++ (char**)&generated_key, PMIX_STRING);
++
++ char *tmp_key;
++ if (PMIX_SUCCESS != rc || NULL == generated_key) {
++ if (NULL != (tmp_key = getenv("OMPI_MCA_orte_precondition_transports"))) {
++ generated_key = strdup(tmp_key);
++ }
++ }
++
+ memset(uu, 0, sizeof(psm2_uuid_t));
+
+ if (!generated_key || (strlen(generated_key) != 33) ||
+@@ -111,10 +127,13 @@
+ "no uuid present", true,
+ generated_key ? "could not be parsed from" :
+ "not present in", ompi_process_info.nodename);
++ free(generated_key);
+ return OMPI_ERROR;
+
+ }
+
++ free(generated_key);
++
+ /* Handle our own errors for opening endpoints */
+ psm2_error_register_handler(ompi_mtl_psm2.ep, ompi_mtl_psm2_errhandler);
+
--
GitLab