From df38e052d0c764f53f6b076a073517c8ff366ffe Mon Sep 17 00:00:00 2001
From: Damian Alvarez <d.alvarez@fz-juelich.de>
Date: Thu, 6 Jan 2022 10:41:40 +0100
Subject: [PATCH] To move to OpenMPI 4.1.2 by default, since it fixes an issue
 with accelerated UD in UCX

---
 Golden_Repo/g/gomkl/gomkl-2021b.eb            |  2 +-
 Golden_Repo/g/gompi/gompi-2021b.eb            |  2 +-
 Golden_Repo/i/iomkl/iomkl-2021b.eb            |  2 +-
 Golden_Repo/i/iompi/iompi-2021b.eb            |  2 +-
 Golden_Repo/n/nvompic/nvompic-2021b.eb        |  2 +-
 .../o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb     | 61 ++++++++++++++++++
 .../o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb    | 61 ++++++++++++++++++
 .../OpenMPI-4.1.2-intel-compilers-2021.4.0.eb | 61 ++++++++++++++++++
 .../o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb     | 62 +++++++++++++++++++
 .../o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb    | 62 +++++++++++++++++++
 .../OpenMPI-4.1.2-intel-compilers-2021.4.0.eb | 62 +++++++++++++++++++
 11 files changed, 374 insertions(+), 5 deletions(-)
 create mode 100644 Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb
 create mode 100644 Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb
 create mode 100644 Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb
 create mode 100644 Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb
 create mode 100644 Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb
 create mode 100644 Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb

diff --git a/Golden_Repo/g/gomkl/gomkl-2021b.eb b/Golden_Repo/g/gomkl/gomkl-2021b.eb
index 4cc078f39..2307330e2 100644
--- a/Golden_Repo/g/gomkl/gomkl-2021b.eb
+++ b/Golden_Repo/g/gomkl/gomkl-2021b.eb
@@ -20,7 +20,7 @@ local_comp_mpi_tc = (local_comp_mpi_tc_name, local_comp_mpi_tc_ver)
 # compiler toolchain dependencies
 dependencies = [
     local_comp,
-    ('OpenMPI', '4.1.1', '', local_comp),  # part of gompi toolchain
+    ('OpenMPI', '4.1.2', '', local_comp),  # part of gompi toolchain
     ('imkl', '2021.4.0', '', local_comp_mpi_tc),
 ]
 
diff --git a/Golden_Repo/g/gompi/gompi-2021b.eb b/Golden_Repo/g/gompi/gompi-2021b.eb
index 86988f972..d522d562f 100644
--- a/Golden_Repo/g/gompi/gompi-2021b.eb
+++ b/Golden_Repo/g/gompi/gompi-2021b.eb
@@ -13,7 +13,7 @@ site_contacts = 'sc@fz-juelich.de'
 toolchain = SYSTEM
 
 local_mpilib = 'OpenMPI'
-local_mpiver = '4.1.1'
+local_mpiver = '4.1.2'
 
 local_compname = 'GCC'
 local_compver = '11.2.0'
diff --git a/Golden_Repo/i/iomkl/iomkl-2021b.eb b/Golden_Repo/i/iomkl/iomkl-2021b.eb
index 6979a5df8..a4dd74df5 100644
--- a/Golden_Repo/i/iomkl/iomkl-2021b.eb
+++ b/Golden_Repo/i/iomkl/iomkl-2021b.eb
@@ -14,7 +14,7 @@ toolchain = SYSTEM
 
 dependencies = [
     ('intel-compilers', local_intelversion),
-    ('OpenMPI', '4.1.1', versionsuffix,
+    ('OpenMPI', '4.1.2', versionsuffix,
      ('intel-compilers', local_intelversion)),
     ('imkl', local_intelversion, versionsuffix, ('iompi', version)),
 ]
diff --git a/Golden_Repo/i/iompi/iompi-2021b.eb b/Golden_Repo/i/iompi/iompi-2021b.eb
index 257c1269b..10b97e3bb 100644
--- a/Golden_Repo/i/iompi/iompi-2021b.eb
+++ b/Golden_Repo/i/iompi/iompi-2021b.eb
@@ -12,7 +12,7 @@ toolchain = SYSTEM
 local_comp_ver = '2021.4.0'
 dependencies = [
     ('intel-compilers', local_comp_ver),
-    ('OpenMPI', '4.1.1', '', ('intel-compilers', local_comp_ver)),
+    ('OpenMPI', '4.1.2', '', ('intel-compilers', local_comp_ver)),
 ]
 
 moduleclass = 'toolchain'
diff --git a/Golden_Repo/n/nvompic/nvompic-2021b.eb b/Golden_Repo/n/nvompic/nvompic-2021b.eb
index 854f8807c..fe3cf8d44 100644
--- a/Golden_Repo/n/nvompic/nvompic-2021b.eb
+++ b/Golden_Repo/n/nvompic/nvompic-2021b.eb
@@ -13,7 +13,7 @@ local_compiler = ('NVHPC', '21.11')
 dependencies = [
     local_compiler,
     ('CUDA', '11.5', '', SYSTEM),
-    ('OpenMPI', '4.1.1', '', local_compiler),
+    ('OpenMPI', '4.1.2', '', local_compiler),
 ]
 
 moduleclass = 'toolchain'
diff --git a/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb b/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb
new file mode 100644
index 000000000..a1495b7f5
--- /dev/null
+++ b/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb
@@ -0,0 +1,61 @@
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'GCC', 'version': '11.2.0'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726'),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'
diff --git a/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb b/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb
new file mode 100644
index 000000000..b1b507116
--- /dev/null
+++ b/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb
@@ -0,0 +1,61 @@
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'NVHPC', 'version': '21.11'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726', '', SYSTEM),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'
diff --git a/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb b/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb
new file mode 100644
index 000000000..5e6c33537
--- /dev/null
+++ b/Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb
@@ -0,0 +1,61 @@
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'intel-compilers', 'version': '2021.4.0'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726'),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'
diff --git a/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb b/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb
new file mode 100644
index 000000000..4e156642b
--- /dev/null
+++ b/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb
@@ -0,0 +1,62 @@
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'GCC', 'version': '11.2.0'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726'),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+# No IME in HDFML
+# configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'
diff --git a/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb b/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb
new file mode 100644
index 000000000..a4a546e98
--- /dev/null
+++ b/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb
@@ -0,0 +1,62 @@
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'NVHPC', 'version': '21.11'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726', '', SYSTEM),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+# No IME in HDFML
+# configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'
diff --git a/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb b/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb
new file mode 100644
index 000000000..00c80316f
--- /dev/null
+++ b/Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb
@@ -0,0 +1,62 @@
+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'intel-compilers', 'version': '2021.4.0'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726'),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+# No IME in HDFML
+# configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'
-- 
GitLab