To move to OpenMPI 4.1.2 by default, since it fixes an issue with

accelerated UD in UCX

Golden_Repo/g/gomkl/gomkl-2021b.eb

@@ -20,7 +20,7 @@ local_comp_mpi_tc = (local_comp_mpi_tc_name, local_comp_mpi_tc_ver)
 # compiler toolchain dependencies
 dependencies = [
     local_comp,
-    ('OpenMPI', '4.1.1', '', local_comp),  # part of gompi toolchain
+    ('OpenMPI', '4.1.2', '', local_comp),  # part of gompi toolchain
     ('imkl', '2021.4.0', '', local_comp_mpi_tc),
 ]
 

Golden_Repo/g/gompi/gompi-2021b.eb

@@ -13,7 +13,7 @@ site_contacts = 'sc@fz-juelich.de'
 toolchain = SYSTEM
 
 local_mpilib = 'OpenMPI'
-local_mpiver = '4.1.1'
+local_mpiver = '4.1.2'
 
 local_compname = 'GCC'
 local_compver = '11.2.0'

Golden_Repo/i/iomkl/iomkl-2021b.eb

@@ -14,7 +14,7 @@ toolchain = SYSTEM
 
 dependencies = [
     ('intel-compilers', local_intelversion),
-    ('OpenMPI', '4.1.1', versionsuffix,
+    ('OpenMPI', '4.1.2', versionsuffix,
      ('intel-compilers', local_intelversion)),
     ('imkl', local_intelversion, versionsuffix, ('iompi', version)),
 ]

Golden_Repo/i/iompi/iompi-2021b.eb

@@ -12,7 +12,7 @@ toolchain = SYSTEM
 local_comp_ver = '2021.4.0'
 dependencies = [
     ('intel-compilers', local_comp_ver),
-    ('OpenMPI', '4.1.1', '', ('intel-compilers', local_comp_ver)),
+    ('OpenMPI', '4.1.2', '', ('intel-compilers', local_comp_ver)),
 ]
 
 moduleclass = 'toolchain'

Golden_Repo/n/nvompic/nvompic-2021b.eb

@@ -13,7 +13,7 @@ local_compiler = ('NVHPC', '21.11')
 dependencies = [
     local_compiler,
     ('CUDA', '11.5', '', SYSTEM),
-    ('OpenMPI', '4.1.1', '', local_compiler),
+    ('OpenMPI', '4.1.2', '', local_compiler),
 ]
 
 moduleclass = 'toolchain'

Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb

+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'GCC', 'version': '11.2.0'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726'),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'

Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb

+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'NVHPC', 'version': '21.11'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726', '', SYSTEM),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'

Golden_Repo/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb

+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'intel-compilers', 'version': '2021.4.0'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726'),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'

Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-GCC-11.2.0.eb

+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'GCC', 'version': '11.2.0'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726'),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+# No IME in HDFML
+# configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'

Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-NVHPC-21.11.eb

+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'NVHPC', 'version': '21.11'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726', '', SYSTEM),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+# No IME in HDFML
+# configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'

Overlays/hdfml_overlay/o/OpenMPI/OpenMPI-4.1.2-intel-compilers-2021.4.0.eb

+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.2'
+
+homepage = 'https://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+toolchain = {'name': 'intel-compilers', 'version': '2021.4.0'}
+toolchainopts = {'pic': True}
+
+source_urls = [
+    'https://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_BZ2]
+checksums = ['9b78c7cf7fc32131c5cf43dd2ab9740149d9d87cadb2e2189f02685749a6b527']
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20210726'),
+    ('pkg-config', '0.29.2'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.5.0'),
+    ('UCX', '1.11.2', '', SYSTEM),
+    ('CUDA', '11.5', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+# No IME in HDFML
+# configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config",
+                                           "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+moduleclass = 'mpi'