OpenMPI made it into 2020

Custom_MNS/flexible_custom_hierarchical_mns.py

@@ -52,7 +52,7 @@ mpi_relevant_versions = {
 }
 
 # MPIs with settings modules
-mpi_with_settings = ['psmpi', 'impi', 'MVAPICH2']
+mpi_with_settings = ['psmpi', 'impi', 'MVAPICH2', 'OpenMPI']
 
 class FlexibleCustomHierarchicalMNS(HierarchicalMNS):
     """Class implementing an example hierarchical module naming scheme."""

Golden_Repo/h/hwloc/hwloc-2.2.0-GCCcore-9.3.0.eb

+easyblock = 'ConfigureMake'
+
+name = 'hwloc'
+version = '2.2.0'
+
+homepage = 'http://www.open-mpi.org/projects/hwloc/'
+
+description = """
+ The Portable Hardware Locality (hwloc) software package provides a portable
+ abstraction (across OS, versions, architectures, ...) of the hierarchical
+ topology of modern architectures, including NUMA memory nodes, sockets, shared
+ caches, cores and simultaneous multithreading. It also gathers various system
+ attributes such as cache and memory information as well as the locality of I/O
+ devices such as network interfaces, InfiniBand HCAs or GPUs. It primarily
+ aims at helping applications with gathering information about modern computing
+ hardware so as to exploit it accordingly and efficiently.
+"""
+
+site_contacts = 'sc@fz-juelich.de'
+
+toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
+
+source_urls = ['https://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/']
+sources = [SOURCE_TAR_GZ]
+
+builddependencies = [
+    ('binutils', '2.34'),
+]
+
+dependencies = [
+    ('numactl', '2.0.13', '', SYSTEM),
+    ('libxml2', '2.9.10'),
+    ('libpciaccess', '0.16'),
+    ('CUDA', '11.0', '', SYSTEM),
+]
+
+configopts = "--enable-libnuma=$EBROOTNUMACTL --enable-cuda --enable-nvml --enable-opencl"
+configopts += "--disable-cairo --disable-gl --disable-libudev "
+
+sanity_check_paths = {
+    'files': ['bin/lstopo', 'include/hwloc/linux.h',
+              'lib/libhwloc.%s' % SHLIB_EXT],
+    'dirs': ['share/man/man3'],
+}
+sanity_check_commands = ['lstopo']
+
+modluafooter = '''
+add_property("arch","gpu")
+'''
+moduleclass = 'system'

Golden_Repo/juwelsbooster_overlay/o/OpenMPI/OpenMPI-4.1.0rc1-GCC-9.3.0.eb

+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.0rc1'
+
+homepage = 'http://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+site_contacts = 'sc@fz-juelich.de'
+
+toolchain = {'name': 'GCC', 'version': '9.3.0'}
+
+source_urls = ['http://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_GZ]
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20200321'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.2.0'),
+    ('UCX', '1.8.1', '', SYSTEM),
+    ('CUDA', '11.0', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+# Without this we might run into this https://github.com/open-mpi/ompi/issues/6058 for some reason. Observed with
+# 4.0.2, but not with 4.0.1
+#preconfigopts = "./autogen.pl --force && "
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+# We don't have yet the RPMs for CentOS8
+# configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config", "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+# To be added to the mpi-settings module:
+#modextravars = {
+#    'SLURM_MPI_TYPE': 'pspmix'
+#}
+#
+## Configure OpenMPI
+#postinstallcmds = [
+#    'echo "mca_base_component_show_load_errors = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "mpi_param_check = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "mpi_show_handle_leaks = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "mpi_warn_on_fork = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "btl = self,vader,openib" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "btl_openib_allow_ib = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "btl_openib_if_include = mlx5_0:1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "bml_r2_show_unreach_errors = 0" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "coll = ^ml" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "coll_hcoll_enable = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "coll_hcoll_np = 0" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "pml = ucx" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "osc = ^rdma" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "opal_abort_print_stack = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "opal_set_max_sys_limits = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "opal_event_include = epoll" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#]
+
+# Add a family for our naming scheme
+modluafooter = '''
+family("mpi")
+add_property("arch","gpu")
+if not ( isloaded("mpi-settings/default") ) then
+    load("mpi-settings/default")
+end
+'''
+
+moduleclass = 'mpi'

Golden_Repo/l/libevent/libevent-2.1.12-GCCcore-9.3.0.eb

+easyblock = 'ConfigureMake'
+
+name = 'libevent'
+version = '2.1.12'
+
+homepage = 'http://libevent.org/'
+description = """The libevent API provides a mechanism to execute a callback function when a specific
+event occurs on a file descriptor or after a timeout has been reached.
+Furthermore, libevent also support callbacks due to signals or regular timeouts.
+"""
+
+site_contacts = 'sc@fz-juelich.de'
+
+toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
+source_urls = ['https://github.com/%(name)s/%(name)s/releases/download/release-%(version)s-stable/']
+sources = ['%(name)s-%(version)s-stable.tar.gz']
+
+builddependencies = [('binutils', '2.34')]
+
+moduleclass = 'lib'

Golden_Repo/o/OpenMPI-settings/OpenMPI-settings-4.1-default.eb

+easyblock = 'SystemBundle'
+
+name = 'OpenMPI-settings'
+version = '4.1'
+versionsuffix = 'default'
+
+homepage = ''
+description = """This is a module to load the default OpenMPI configuration"""
+
+site_contacts = 'd.alvarez@fz-juelich.de'
+
+toolchain = SYSTEM
+
+source_urls = []
+
+sources = []
+modextravars = {
+    'OMPI_MCA_mca_base_component_show_load_errors': '1',
+    'OMPI_MCA_mpi_param_check': '1',
+    'OMPI_MCA_mpi_show_handle_leaks': '1',
+    'OMPI_MCA_mpi_warn_on_fork': '1',
+    'OMPI_MCA_btl': 'self,vader,openib',
+    'OMPI_MCA_btl_openib_allow_ib': '1',
+    'OMPI_MCA_bml_r2_show_unreach_errors': '0',
+    'OMPI_MCA_coll': '^ml',
+    'OMPI_MCA_coll_hcoll_enable': '1',
+    'OMPI_MCA_coll_hcoll_np': '0',
+    'OMPI_MCA_pml': 'ucx',
+    'OMPI_MCA_osc': '^rdma',
+    'OMPI_MCA_opal_abort_print_stack': '1',
+    'OMPI_MCA_opal_set_max_sys_limits': '1',
+    'OMPI_MCA_opal_event_include': 'epoll',
+}
+
+moduleclass = 'system'

Golden_Repo/o/OpenMPI/OpenMPI-4.1.0rc1-GCC-9.3.0.eb

+easyblock = 'ConfigureMake'
+
+name = 'OpenMPI'
+version = '4.1.0rc1'
+
+homepage = 'http://www.open-mpi.org/'
+description = """The Open MPI Project is an open source MPI-3 implementation."""
+
+site_contacts = 'sc@fz-juelich.de'
+
+toolchain = {'name': 'GCC', 'version': '9.3.0'}
+
+source_urls = ['http://www.open-mpi.org/software/ompi/v%(version_major_minor)s/downloads']
+sources = [SOURCELOWER_TAR_GZ]
+
+osdependencies = [
+    # needed for --with-verbs
+    ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'),
+    # needed for --with-pmix
+    ('pmix-devel'),
+]
+
+builddependencies = [
+    ('Autotools', '20200321'),
+]
+
+dependencies = [
+    ('zlib', '1.2.11'),
+    ('hwloc', '2.2.0'),
+    ('UCX', '1.8.1', '', SYSTEM),
+    ('CUDA', '11.0', '', SYSTEM),
+    ('libevent', '2.1.12'),
+]
+
+# Without this we might run into this https://github.com/open-mpi/ompi/issues/6058 for some reason. Observed with
+# 4.0.2, but not with 4.0.1
+#preconfigopts = "./autogen.pl --force && "
+
+configopts = '--enable-shared '
+configopts += '--with-hwloc=$EBROOTHWLOC '  # hwloc support
+configopts += '--with-ucx=$EBROOTUCX '
+configopts += '--with-verbs '
+configopts += '--with-libevent=$EBROOTLIBEVENT '
+configopts += '--without-orte '
+configopts += '--without-psm2 '
+configopts += '--disable-oshmem '
+configopts += '--with-cuda=$EBROOTCUDA '
+configopts += '--with-ime=/opt/ddn/ime '
+configopts += '--with-gpfs '
+
+# to enable SLURM integration (site-specific)
+configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte'
+
+local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"]
+sanity_check_paths = {
+    'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] +
+             ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] +
+             ["include/%s.h" % x for x in ["mpi-ext", "mpif-config", "mpif", "mpi", "mpi_portable_platform"]],
+    'dirs': [],
+}
+
+# To be added to the mpi-settings module:
+#modextravars = {
+#    'SLURM_MPI_TYPE': 'pspmix'
+#}
+#
+## Configure OpenMPI
+#postinstallcmds = [
+#    'echo "mca_base_component_show_load_errors = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "mpi_param_check = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "mpi_show_handle_leaks = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "mpi_warn_on_fork = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "btl = self,vader,openib" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "btl_openib_allow_ib = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "btl_openib_if_include = mlx5_0:1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "bml_r2_show_unreach_errors = 0" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "coll = ^ml" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "coll_hcoll_enable = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "coll_hcoll_np = 0" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "pml = ucx" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "osc = ^rdma" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "opal_abort_print_stack = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "opal_set_max_sys_limits = 1" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#    'echo "opal_event_include = epoll" >> %(installdir)s/etc/openmpi-mca-params.conf',
+#]
+
+# Add a family for our naming scheme
+modluafooter = '''
+family("mpi")
+add_property("arch","gpu")
+if not ( isloaded("mpi-settings/default") ) then
+    load("mpi-settings/default")
+end
+'''
+
+moduleclass = 'mpi'