diff --git a/Overlays/jurecadc_overlay/b/BullMPI-settings/BullMPI-settings-4.1-CUDA.eb b/Overlays/jurecadc_overlay/b/BullMPI-settings/BullMPI-settings-4.1-CUDA.eb new file mode 100644 index 0000000000000000000000000000000000000000..08c35eea478aa9ab34ac94beffeaaca35fa07f13 --- /dev/null +++ b/Overlays/jurecadc_overlay/b/BullMPI-settings/BullMPI-settings-4.1-CUDA.eb @@ -0,0 +1,44 @@ +easyblock = 'SystemBundle' + +name = 'BullMPI-settings' +version = '4.1' +versionsuffix = 'CUDA' + +homepage = '' +description = """This is a module to load the default BullMPI configuration""" + +site_contacts = 'd.alvarez@fz-juelich.de' + +toolchain = SYSTEM + +source_urls = [] + +sources = [] +modextravars = { + 'SLURM_MPI_TYPE': 'pspmix', + 'UCX_TLS': 'rc_x,cuda_ipc,gdr_copy,self,sm,cuda_copy', + 'UCX_MEMTYPE_CACHE': 'n', + 'OMPI_MCA_mca_base_component_show_load_errors': '1', + 'OMPI_MCA_mpi_param_check': '1', + 'OMPI_MCA_mpi_show_handle_leaks': '1', + 'OMPI_MCA_mpi_warn_on_fork': '1', + # Disable uct for the time being due to: + # https://github.com/openucx/ucx/wiki/BullMPI-and-OpenSHMEM-installation-with-UCX#running-open-mpi-with-ucx + # Also openib, since it is deprecated and should be substituted by the UCX support in the pml + 'OMPI_MCA_btl': '^uct,openib', + 'OMPI_MCA_btl_openib_allow_ib': '1', + 'OMPI_MCA_bml_r2_show_unreach_errors': '0', + 'OMPI_MCA_coll': '^ml', + 'OMPI_MCA_coll_hcoll_enable': '1', + 'OMPI_MCA_coll_hcoll_np': '0', + 'OMPI_MCA_pml': 'ucx', + 'OMPI_MCA_osc': '^rdma', + 'OMPI_MCA_opal_abort_print_stack': '1', + 'OMPI_MCA_opal_set_max_sys_limits': '1', + 'OMPI_MCA_opal_event_include': 'epoll', + 'OMPI_MCA_btl_openib_warn_default_gid_prefix': '0', + # OMPIO does not seem to work reliably on our system + 'OMPI_MCA_io': 'romio321', +} + +moduleclass = 'system' diff --git a/Overlays/jurecadc_overlay/b/BullMPI/BullMPI-4.1.0-GCC-11.2.0.eb b/Overlays/jurecadc_overlay/b/BullMPI/BullMPI-4.1.0-GCC-11.2.0.eb new file mode 100644 index 0000000000000000000000000000000000000000..dd8f3d40d61766f59ef5f8a0da5425cc16b64f8b --- /dev/null +++ b/Overlays/jurecadc_overlay/b/BullMPI/BullMPI-4.1.0-GCC-11.2.0.eb @@ -0,0 +1,72 @@ +easyblock = 'ConfigureMake' + +name = 'BullMPI' +version = '4.1.0' + +homepage = 'https://www.open-mpi.org/' +description = """BullMPI is an MPI runtime based on OpenMPI with specific optimizations""" + +toolchain = {'name': 'GCC', 'version': '11.2.0'} +toolchainopts = {'pic': True} + +sources = [f'{name}_{version}.tar'] +checksums = ['e08cce34ce6388bd7c8203a283f48c344f25044c9efbff6a27e5f1e0cbd01b14'] + +osdependencies = [ + # needed for --with-verbs + ('libibverbs-dev', 'libibverbs-devel', 'rdma-core-devel'), + # needed for --with-pmix + ('pmix-devel'), +] + +builddependencies = [ + ('Autotools', '20210726', '', SYSTEM), + ('pkg-config', '0.29.2'), +] + +dependencies = [ + ('zlib', '1.2.11'), + ('hwloc', '2.5.0'), + ('UCX', '1.11.2', '', SYSTEM), + ('CUDA', '11.4', '', SYSTEM), + ('libevent', '2.1.12'), +] + +start_dir = f'openmpi-{version}' + +unpack_options = f'&& rpm2cpio openmpi-bull-gnu-{version}-1.Bull.2.0.src.rpm | cpio -idmv && ' +unpack_options += f'tar zxvf openmpi-{version}.tar.gz' + +# We need to remove -march=native from CFLAGS, otherwise the compilation fails when trying to compile the op:avx +# component, due to lack of avx512 support. avx512 is safe to enable, even on non-avx512 architectures, since it is used +# in just one file, and the execution of that code is determined at runtime, depending on the processor ISA +preconfigopts = 'CFLAGS="-O2 -ftree-vectorize -fno-math-errno -fPIC" ' +prebuildopts = preconfigopts + +# General OpenMPI options +configopts = '--enable-shared ' +configopts += '--with-hwloc=$EBROOTHWLOC ' # hwloc support +configopts += '--with-ucx=$EBROOTUCX ' +configopts += '--with-verbs ' +configopts += '--with-libevent=$EBROOTLIBEVENT ' +configopts += '--without-orte ' +configopts += '--without-psm2 ' +configopts += '--disable-oshmem ' +configopts += '--with-cuda=$EBROOTCUDA ' +configopts += '--with-ime=/opt/ddn/ime ' +configopts += '--with-gpfs ' +configopts += '--with-hcoll=$(pkg-config --variable prefix hcoll) ' + +# to enable SLURM integration (site-specific) +configopts += '--with-slurm --with-pmix=external --with-libevent=external --with-ompi-pmix-rte' + +local_libs = ["mpi_mpifh", "mpi", "ompitrace", "open-pal", "open-rte"] +sanity_check_paths = { + 'files': ["bin/%s" % local_binfile for local_binfile in ["ompi_info", "opal_wrapper"]] + + ["lib/lib%s.%s" % (local_libfile, SHLIB_EXT) for local_libfile in local_libs] + + ["include/%s.h" % x for x in ["mpi-ext", "mpif-config", + "mpif", "mpi", "mpi_portable_platform"]], + 'dirs': [], +} + +moduleclass = 'mpi'