Skip to content
Snippets Groups Projects
Commit 296a1bbd authored by Damian Alvarez's avatar Damian Alvarez
Browse files

First 2020 packages, from scratch up to GCC+ParaStationMPI

parent 82905059
Branches
No related tags found
No related merge requests found
Showing
with 1228 additions and 0 deletions
##
# Copyright 2017 Forschungszentrum Juelich GmbH
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# http://github.com/hpcugent/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
@author: Damian Alvarez (Forschungszentrum Juelich GmbH)
"""
from easybuild.easyblocks.generic.bundle import Bundle
class SystemBundle(Bundle):
"""
Support for creating a bundle that allows to prepend absolute paths in LD_LIBRARY_PATH
"""
def make_module_extra(self):
"""Prepend variables allowing absolute paths, and proceed as normal"""
lines = ['']
modextrapaths = self.cfg['modextrapaths']
for (key, value) in self.cfg['modextrapaths'].items():
if isinstance(value, basestring):
value = [value]
elif not isinstance(value, (tuple, list)):
raise EasyBuildError("modextrapaths dict value %s (type: %s) is not a list or tuple",
value, type(value))
lines.append(self.module_generator.prepend_paths(key, value, allow_abs=True))
self.cfg['modextrapaths'] = {}
txt = super(SystemBundle, self).make_module_extra()
self.cfg['modextrapaths'] = modextrapaths
txt += ''.join(lines)
return txt
##
# Copyright 2009-2019 Ghent University, Forschungszentrum Juelich
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# the Hercules foundation (http://www.herculesstichting.be/in_English)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for building and installing the MPICH MPI library and derivatives, implemented as an easyblock
@author: Stijn De Weirdt (Ghent University)
@author: Dries Verdegem (Ghent University)
@author: Kenneth Hoste (Ghent University)
@author: Pieter De Baets (Ghent University)
@author: Jens Timmerman (Ghent University)
@author: Damian Alvarez (Forschungszentrum Juelich)
@author: Xavier Besseron (University of Luxembourg)
"""
import os
from distutils.version import LooseVersion
import easybuild.tools.environment as env
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.systemtools import get_shared_lib_ext
class EB_MPICH(ConfigureMake):
"""
Support for building the MPICH MPI library and derivatives.
- basically redefinition of environment variables
"""
@staticmethod
def extra_options(extra_vars=None):
"""Define custom easyconfig parameters specific to MPICH."""
extra_vars = ConfigureMake.extra_options(extra_vars)
extra_vars.update({
'debug': [False, "Enable debug build (which is slower)", CUSTOM],
})
return extra_vars
# MPICH configure script complains when F90 or F90FLAGS are set,
# they should be replaced with FC/FCFLAGS instead.
# Additionally, there are a set of variables (FCFLAGS among them) that should not be set at configure time,
# or they will leak in the mpix wrappers.
# Specific variables to be included in the wrapper exists, but they changed between MPICH 3.1.4 and MPICH 3.2
# and in a typical scenario we probably don't want them.
def correct_mpich_build_env(self):
"""
Method to correctly set the environment for MPICH and derivatives
"""
env_vars = ['CFLAGS', 'CPPFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'LDFLAGS', 'LIBS']
vars_to_unset = ['F90', 'F90FLAGS']
for envvar in env_vars:
envvar_val = os.getenv(envvar)
if envvar_val:
new_envvar = 'MPICHLIB_%s' % envvar
new_envvar_val = os.getenv(new_envvar)
vars_to_unset.append(envvar)
if envvar_val == new_envvar_val:
self.log.debug("$%s == $%s, just defined $%s as empty", envvar, new_envvar, envvar)
elif new_envvar_val is None:
env.setvar(new_envvar, envvar_val)
else:
raise EasyBuildError("Both $%s and $%s set, can I overwrite $%s with $%s (%s) ?",
envvar, new_envvar, new_envvar, envvar, envvar_val)
env.unset_env_vars(vars_to_unset)
def add_mpich_configopts(self):
"""
Method to add common configure options for MPICH-based MPI libraries
"""
# additional configuration options
add_configopts = []
# use POSIX threads
add_configopts.append('--with-thread-package=pthreads')
if self.cfg['debug']:
# debug build, with error checking, timing and debug info
# note: this will affect performance
add_configopts.append('--enable-fast=none')
else:
# optimized build, no error checking, timing or debug info
add_configopts.append('--enable-fast')
# enable shared libraries, using GCC and GNU ld options
add_configopts.extend(['--enable-shared', '--enable-sharedlibs=gcc'])
# enable static libraries
add_configopts.extend(['--enable-static'])
# enable Fortran 77/90 and C++ bindings
add_configopts.extend(['--enable-f77', '--enable-fc', '--enable-cxx'])
self.cfg.update('configopts', ' '.join(add_configopts))
def configure_step(self, add_mpich_configopts=True):
"""
Custom configuration procedure for MPICH
* add common configure options for MPICH-based MPI libraries
* unset environment variables that leak into mpi* wrappers, and define $MPICHLIB_* equivalents instead
"""
# things might go wrong if a previous install dir is present, so let's get rid of it
if not self.cfg['keeppreviousinstall']:
self.log.info("Making sure any old installation is removed before we start the build...")
super(EB_MPICH, self).make_dir(self.installdir, True, dontcreateinstalldir=True)
if add_mpich_configopts:
self.add_mpich_configopts()
self.correct_mpich_build_env()
super(EB_MPICH, self).configure_step()
# make and make install are default
def sanity_check_step(self, custom_paths=None, use_new_libnames=None, check_launchers=True, check_static_libs=True):
"""
Custom sanity check for MPICH
"""
shlib_ext = get_shared_lib_ext()
if custom_paths is None:
custom_paths = {}
if use_new_libnames is None:
# cfr. http://git.mpich.org/mpich.git/blob_plain/v3.1.1:/CHANGES
# MPICH changed its library names sinceversion 3.1.1
use_new_libnames = LooseVersion(self.version) >= LooseVersion('3.1.1')
# Starting MPICH 3.1.1, libraries have been renamed
# cf http://git.mpich.org/mpich.git/blob_plain/v3.1.1:/CHANGES
if use_new_libnames:
libnames = ['mpi', 'mpicxx', 'mpifort']
else:
libnames = ['fmpich', 'mpichcxx', 'mpichf90', 'mpich', 'mpl', 'opa']
binaries = ['mpicc', 'mpicxx', 'mpif77', 'mpif90']
if check_launchers:
binaries.extend(['mpiexec', 'mpiexec.hydra', 'mpirun'])
bins = [os.path.join('bin', x) for x in binaries]
headers = [os.path.join('include', x) for x in ['mpi.h', 'mpicxx.h', 'mpif.h']]
lib_exts = [shlib_ext]
if check_static_libs:
lib_exts.append('a')
libs_fn = ['lib%s.%s' % (l, e) for l in libnames for e in lib_exts]
libs = [(os.path.join('lib', l), os.path.join('lib64', l)) for l in libs_fn]
custom_paths.setdefault('dirs', []).extend(['bin', 'include', ('lib', 'lib64')])
custom_paths.setdefault('files', []).extend(bins + headers + libs)
super(EB_MPICH, self).sanity_check_step(custom_paths=custom_paths)
##
# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild
#
# Copyright:: Copyright 2019 Forschungszentrum Juelich GmbH
# Authors:: Damian Alvarez
# License:: MIT/GPL
# $Id$
##
"""
EasyBuild support for NVIDIA libs, implemented as an easyblock
@author: Damian Alvarez (Forschungszentrum Juelich)
"""
import os
from distutils.version import LooseVersion
from easybuild.easyblocks.generic.binary import Binary
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.filetools import copy, expand_glob_paths, mkdir
from easybuild.tools.run import run_cmd
from easybuild.tools.systemtools import get_shared_lib_ext
class EB_nvidia_minus_driver(Binary):
"""
Support for installing NVIDIA libs.
"""
@staticmethod
def extra_options():
"""Support for generic 'default' modules with specific real versions"""
extra_vars = {
'realversion': [None, "Real version to be used when version = 'default'", CUSTOM],
}
return extra_vars
def extract_step(self):
"""Extract installer to have more control, e.g. options, patching Perl scripts, etc."""
if self.cfg['realversion']:
version = self.cfg['realversion']
else:
version = self.version
if LooseVersion(version) < LooseVersion('400.00'):
run_files_dir = 'run_files'
else:
run_files_dir = 'builds'
# Source comes from the nvidia drivers
if self.src[0]['name'].lower().startswith("nvidia"):
execpath = self.src[0]['path']
# Source comes from the CUDA toolkit
else:
# Decompress initial file
execpath = self.src[0]['path']
run_cmd("/bin/sh " + execpath + " --noexec --nox11 --target " + self.builddir)
execpath = os.path.join(self.builddir, run_files_dir, 'NVIDIA-Linux-x86_64-%s.run' % version)
# Decompress file containing libraries
self.libsdir = os.path.join(self.builddir, "nvidia-libs")
run_cmd("/bin/sh " + execpath + " -x --target " + self.libsdir)
def install_step(self):
"Install NVIDIA libs simply by copying files. We can't user the installer because it requires root privileges."
# list of libs
libs = expand_glob_paths([os.path.join(self.libsdir, 'lib*.so*')])
libs += expand_glob_paths([os.path.join(self.libsdir, '*.la')])
libs += [os.path.join(self.libsdir, 'nvidia_drv.so')]
# list of binaries
binaries = ['nvidia-bug-report.sh',
'nvidia-cuda-mps-control',
'nvidia-cuda-mps-server',
'nvidia-debugdump',
'nvidia-settings',
'nvidia-smi',
'nvidia-xconfig']
binaries = [os.path.join(self.libsdir, x) for x in binaries]
# list of manpages
manpages = ['nvidia-settings.1.gz',
'nvidia-cuda-mps-control.1.gz',
'nvidia-xconfig.1.gz',
'nvidia-smi.1.gz']
manpages = [os.path.join(self.libsdir, x) for x in manpages]
copy(libs, os.path.join(self.installdir, 'lib64'))
copy(binaries, os.path.join(self.installdir, 'bin'))
copy(manpages, os.path.join(self.installdir, 'man', 'man1'))
def post_install_step(self):
"""Generate the appropriate symlinks"""
libdir = os.path.join(self.installdir, 'lib64')
# Run ldconfig to create missing symlinks (libcuda.so.1, etc)
run_cmd("ldconfig -N %s" % libdir)
# Create an extra symlink for libcuda.so, otherwise PGI 19.X breaks
# Create an extra symlink for libnvidia-ml.so, otherwise MVAPICH2 doesn't find it if it doesn't rely on stubs
missing_links = ['libcuda.so', 'libnvidia-ml.so']
for missing_link in missing_links:
run_cmd("ln -s %s/%s.1 %s/%s" % (libdir, missing_link, libdir, missing_link))
super(EB_nvidia_minus_driver, self).post_install_step()
def sanity_check_step(self):
"""Custom sanity check for NVIDIA libs."""
shlib_ext = get_shared_lib_ext()
chk_libdir = ["lib64"]
nvlibs = ["cuda", "nvidia-ml"]
custom_paths = {
'files': [os.path.join("bin", x) for x in ["nvidia-smi"]] +
[os.path.join("%s", "lib%s.%s.1") % (x, y, shlib_ext) for x in chk_libdir for y in nvlibs],
'dirs': [''],
}
super(EB_nvidia_minus_driver, self).sanity_check_step(custom_paths=custom_paths)
##
# Copyright 2016-2018 Ghent University, Forschungszentrum Juelich
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for building and installing the ParaStationMPI library, implemented as an easyblock
@author: Damian Alvarez (Forschungszentrum Juelich)
"""
import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
import os
import re
from distutils.version import LooseVersion
from easybuild.easyblocks.mpich import EB_MPICH
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.filetools import mkdir, remove_dir, write_file
from easybuild.tools.modules import get_software_root
from easybuild.tools.run import run_cmd
PINGPONG_PGO_TEST = """
/****************************************
* Potentially buggy, ugly and extremely simple MPI ping pong.
* The goal is to trigger pscom internal routines to generate
* profiles to be used by PGO-enabled compilers.
*
* Nothing more, nothing less.
*
* We try small and large messages to walk the path for eager
* and rendezvous protocols
*
* author: Damian Alvarez (d.alvarez@fz-juelich.de)
****************************************/
#include "mpi.h"
#define MAX_LENGTH 1048576
#define ITERATIONS 1000
int main(int argc, char *argv[]){
int myid, vlength, i, err, eslength;
MPI_Request requests[2];
char error_string[MPI_MAX_ERROR_STRING];
double v0[MAX_LENGTH], v1[MAX_LENGTH];
MPI_Status stat;
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &myid);
// Test blocking point to point
for (vlength=1; vlength<=MAX_LENGTH; vlength*=2){
for (i=0; i<ITERATIONS; i++){
MPI_Barrier(MPI_COMM_WORLD);
// Not really needed, but it might be interesting to PGO this for benchmarking?
MPI_Wtime();
if (myid == 0){
err = MPI_Send(v0, vlength*sizeof(double), MPI_BYTE, 1, 0, MPI_COMM_WORLD);
MPI_Error_string(err, error_string, &eslength);
err = MPI_Recv(v1, vlength*sizeof(double), MPI_BYTE, 1, 0, MPI_COMM_WORLD, &stat);
MPI_Error_string(err, error_string, &eslength);
}
else{
err = MPI_Recv(v1, vlength*sizeof(double), MPI_BYTE, 0, 0, MPI_COMM_WORLD, &stat);
MPI_Error_string(err, error_string, &eslength);
err = MPI_Send(v0, vlength*sizeof(double), MPI_BYTE, 0, 0, MPI_COMM_WORLD);
MPI_Error_string(err, error_string, &eslength);
}
MPI_Wtime();
}
}
// Test non-blocking point to point
for (vlength=1; vlength<=MAX_LENGTH; vlength*=2){
for (i=0; i<ITERATIONS; i++){
if (myid == 0){
err = MPI_Isend(v0, vlength*sizeof(double), MPI_BYTE, 1, 0, MPI_COMM_WORLD, &requests[0]);
MPI_Error_string(err, error_string, &eslength);
err = MPI_Irecv(v1, vlength*sizeof(double), MPI_BYTE, 1, 0, MPI_COMM_WORLD, &requests[1]);
MPI_Error_string(err, error_string, &eslength);
}
else{
err = MPI_Irecv(v1, vlength*sizeof(double), MPI_BYTE, 0, 0, MPI_COMM_WORLD, &requests[1]);
MPI_Error_string(err, error_string, &eslength);
err = MPI_Isend(v0, vlength*sizeof(double), MPI_BYTE, 0, 0, MPI_COMM_WORLD, &requests[0]);
MPI_Error_string(err, error_string, &eslength);
}
MPI_Waitall(2, &requests[0], MPI_STATUSES_IGNORE);
}
}
MPI_Finalize();
}
"""
class EB_psmpi(EB_MPICH):
"""
Support for building the ParaStationMPI library.
* Determines the compiler to be used based on the toolchain
* Enables threading if required by the easyconfig
* Sets extra MPICH options if required by the easyconfig
* Generates a PGO profile and uses it if required
"""
def __init__(self, *args, **kwargs):
super(EB_psmpi, self).__init__(*args, **kwargs)
self.profdir = os.path.join(self.builddir, 'profile')
@staticmethod
def extra_options(extra_vars=None):
"""Define custom easyconfig parameters specific to ParaStationMPI."""
extra_vars = EB_MPICH.extra_options(extra_vars)
# ParaStationMPI doesn't offer this build option, and forcing it in the MPICH build
# can be potentially conflictive with other options set by psmpi configure script.
del extra_vars['debug']
extra_vars.update({
'mpich_opts': [None, "Optional options to configure MPICH", CUSTOM],
'threaded': [False, "Enable multithreaded build (which is slower)", CUSTOM],
'pscom_allin_path': [None, "Enable pscom integration by giving its source path", CUSTOM],
'pgo': [False, "Enable profiling guided optimizations", CUSTOM],
'mpiexec_cmd': ['srun -n ', "Command to run benchmarks to generate PGO profile. With -n switch", CUSTOM],
'cuda': [False, "Enable CUDA awareness", CUSTOM],
})
return extra_vars
def pgo_steps(self):
"""
Set of steps to be performed after the initial installation, if PGO were enabled
"""
self.log.info("Running PGO steps...")
# Remove old profiles
remove_dir(self.profdir)
mkdir(self.profdir)
# Clean the old build
run_cmd('make distclean')
# Compile and run example to generate profile
print_msg("generating PGO profile...")
(out, _) = run_cmd('%s 2 hostname' % self.cfg['mpiexec_cmd'])
nodes = out.split()
if nodes[0] == nodes[1]:
raise EasyBuildError("The profile is generated with 1 node! Use 2 nodes to generate a proper profile!")
write_file('pingpong.c', PINGPONG_PGO_TEST)
run_cmd('%s/bin/mpicc pingpong.c -o pingpong' % self.installdir)
run_cmd('PSP_SHM=0 %s 2 pingpong' % self.cfg['mpiexec_cmd'])
# Check that the profiles are there
new_profs = os.listdir(self.profdir)
if not new_profs:
raise EasyBuildError("The PGO profiles where not found in the expected directory (%s)" % self.profdir)
# Change PGO related options
self.cfg['pgo'] = False
self.cfg['configopts'] = re.sub('--with-profile=gen', '--with-profile=use', self.cfg['configopts'])
# Reconfigure
print_msg("configuring with PGO...")
self.log.info("Running configure_step with PGO...")
self.configure_step()
# Rebuild
print_msg("building with PGO...")
self.log.info("Running build_step with PGO...")
self.build_step()
# Reinstall
print_msg("installing with PGO...")
self.log.info("Running install_step with PGO...")
self.install_step()
# MPICH configure script complains when F90 or F90FLAGS are set,
def configure_step(self):
"""
Custom configuration procedure for ParaStationMPI.
* Sets the correct options
* Calls the MPICH configure_step, disabling the default MPICH options
"""
comp_opts = {
toolchain.GCC: 'gcc',
toolchain.INTELCOMP: 'intel',
toolchain.PGI: 'pgi',
}
# ParaStationMPI defines its environment through confsets. So these should be unset
env_vars = ['CFLAGS', 'CPPFLAGS', 'CXXFLAGS', 'FCFLAGS', 'FFLAGS', 'LDFLAGS', 'LIBS']
env.unset_env_vars(env_vars)
self.log.info("Unsetting the following variables: " + ' '.join(env_vars))
# Enable CUDA
if self.cfg['cuda']:
self.log.info("Enabling CUDA-Awareness...")
self.cfg.update('configopts', ' --with-cuda')
# Set confset
comp_fam = self.toolchain.comp_family()
if comp_fam in comp_opts:
self.cfg.update('configopts', ' --with-confset=%s' % comp_opts[comp_fam])
else:
raise EasyBuildError("Compiler %s not supported. Valid options are: %s",
comp_fam, ', '.join(comp_opts.keys()))
# Enable threading, if necessary
if self.cfg['threaded']:
self.cfg.update('configopts', ' --with-threading')
# Add extra mpich options, if any
if self.cfg['mpich_opts'] is not None:
self.cfg.update('configopts', ' --with-mpichconf="%s"' % self.cfg['mpich_opts'])
# Add PGO related options, if enabled
if self.cfg['pgo']:
self.cfg.update('configopts', ' --with-profile=gen --with-profdir=%s' % self.profdir)
# Lastly, set pscom related variables
if self.cfg['pscom_allin_path'] is None:
pscom_path = get_software_root('pscom')
else:
pscom_path = self.cfg['pscom_allin_path'].strip()
self.cfg.update('configopts', ' --with-pscom-allin="%s"' % pscom_path)
pscom_flags = 'export PSCOM_LDFLAGS="-L{0}/lib $PSCOM_LDFLAGS" &&'.format(pscom_path)
pscom_flags += ' export PSCOM_CPPFLAGS="-I{0}/include $PSCOM_CPPFLAGS" &&'.format(pscom_path)
self.cfg.update('preconfigopts', pscom_flags)
super(EB_psmpi, self).configure_step(add_mpich_configopts=False)
# If PGO is enabled, install, generate a profile, and start over
def install_step(self):
"""
Custom installation procedure for ParaStationMPI.
* If PGO is requested, installs, generate a profile, and start over
"""
super(EB_psmpi, self).install_step()
if self.cfg['pgo']:
self.pgo_steps()
def sanity_check_step(self):
"""
Disable the checking of the launchers for ParaStationMPI
"""
# cfr. http://git.mpich.org/mpich.git/blob_plain/v3.1.1:/CHANGES
# MPICH changed its library names sinceversion 3.1.1.
# cfr. https://github.com/ParaStation/psmpi2/blob/master/ChangeLog
# ParaStationMPI >= 5.1.1-1 is based on MPICH >= 3.1.3.
# ParaStationMPI < 5.1.1-1 is based on MPICH < 3.1.1.
use_new_libnames = LooseVersion(self.version) >= LooseVersion('5.1.1-1')
super(EB_psmpi, self).sanity_check_step(use_new_libnames=use_new_libnames, check_launchers=False, check_static_libs=False)
The table below shows the details of the toolchains in the 2020 stage:
- Base
| Toolchain name | Toolchain version | Underlying GCC | Compiler | MPI | CUDA | Math libraries | Includes software from |
|----------------|---------------------------|----------------|------------------|------------------------|----------|----------------|---------------------------|
| GCCcore | 10.1.0 | 10.1.0 | | | | | |
- Compilers
| Toolchain name | Toolchain version | Underlying GCC | Compiler | MPI | CUDA | Math libraries | Includes software from |
|----------------|---------------------------|----------------|------------------|------------------------|----------|----------------|---------------------------|
| GCC | 10.1.0 | 10.1.0 | GCC 10.1.0 | | | | GCCcore |
- Compilers+MPI
| Toolchain name | Toolchain version | Underlying GCC | Compiler | MPI | CUDA | Math libraries | Includes software from |
|----------------|---------------------------|----------------|------------------|------------------------|----------|----------------|---------------------------|
| gpsmpi | 2020 | 10.1.0 | GCC 10.1.0 | ParaStationMPI 5.4.X | | | GCCcore, GCC |
- Compilers+MPI+Math
| Toolchain name | Toolchain version | Underlying GCC | Compiler | MPI | CUDA | Math libraries | Includes software from |
|----------------|---------------------------|----------------|------------------|------------------------|----------|----------------|---------------------------|
| gpsmkl | 2020 | 10.1.0 | GCC 10.1.0 | ParaStationMPI 5.4.X | | MKL 2020.1.217 | GCCcore, GCC, gpsmpi |
easyblock = 'ConfigureMake'
name = 'Autoconf'
version = '2.69'
homepage = 'http://www.gnu.org/software/autoconf/'
description = """Autoconf is an extensible package of M4 macros that produce shell scripts
to automatically configure software source code packages. These scripts can adapt the
packages to many kinds of UNIX-like systems without manual user intervention. Autoconf
creates a configuration script for a package from a template file that lists the
operating system features that the package can use, in the form of M4 macro calls.
"""
site_contacts = 'sc@fz-juelich.de'
toolchain = SYSTEM
source_urls = [GNU_SOURCE]
sources = [SOURCELOWER_TAR_GZ]
builddependencies = [
('binutils', '2.34')
]
dependencies = [
('M4', '1.4.18'),
]
sanity_check_paths = {
'files': ["bin/%s" % x for x in ["autoconf", "autoheader", "autom4te", "autoreconf", "autoscan",
"autoupdate", "ifnames"]],
'dirs': [],
}
moduleclass = 'devel'
##
# This file is an EasyBuild reciPY as per https://github.com/hpcugent/easybuild
#
# Copyright:: Copyright 2012-2014 Uni.Lu/LCSB, NTUA
# Authors:: Fotis Georgatos <fotis@cern.ch>
# License:: MIT/GPL
# $Id$
#
# This work implements a part of the HPCBIOS project and is a component of the policy:
# http://hpcbios.readthedocs.org/en/latest/
##
easyblock = 'ConfigureMake'
name = 'Automake'
version = "1.16.2"
homepage = 'http://www.gnu.org/software/automake/automake.html'
description = """
Automake: GNU Standards-compliant Makefile generator
"""
site_contacts = 'sc@fz-juelich.de'
toolchain = SYSTEM
source_urls = [GNU_SOURCE]
sources = [SOURCELOWER_TAR_GZ]
builddependencies = [('binutils', '2.34')]
dependencies = [('Autoconf', '2.69')]
sanity_check_paths = {
'files': ['bin/automake', 'bin/aclocal'],
'dirs': []
}
moduleclass = 'devel'
easyblock = 'Bundle'
name = 'Autotools'
version = '20200321' # date of the most recent change
homepage = 'http://autotools.io'
description = """
This bundle collect the standard GNU build tools: Autoconf, Automake and libtool
"""
site_contacts = 'sc@fz-juelich.de'
toolchain = SYSTEM
dependencies = [
('Autoconf', '2.69'), # 20120424
('Automake', '1.16.2'), # 20200321
('libtool', '2.4.6'), # 20150215
]
moduleclass = 'devel'
easyblock = 'ConfigureMake'
name = 'Bison'
version = '3.6.4'
homepage = 'http://www.gnu.org/software/bison'
description = """Bison is a general-purpose parser generator that converts an annotated context-free grammar
into a deterministic LR or generalized LR (GLR) parser employing LALR(1) parser tables.
"""
site_contacts = 'sc@fz-juelich.de'
toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
sources = [SOURCELOWER_TAR_GZ]
source_urls = [GNU_SOURCE]
builddependencies = [
('M4', '1.4.18'),
# use same binutils version that was used when building GCCcore toolchain
('binutils', '2.34', '', True),
]
sanity_check_paths = {
'files': ["bin/%s" % x for x in ["bison", "yacc"]] + ["lib/liby.a"],
'dirs': [],
}
moduleclass = 'lang'
easyblock = 'ConfigureMake'
name = 'Bison'
version = '3.6.4'
homepage = 'http://www.gnu.org/software/bison'
description = """Bison is a general-purpose parser generator that converts an annotated context-free grammar
into a deterministic LR or generalized LR (GLR) parser employing LALR(1) parser tables.
"""
site_contacts = 'sc@fz-juelich.de'
toolchain = SYSTEM
sources = [SOURCELOWER_TAR_GZ]
source_urls = [GNU_SOURCE]
builddependencies = [('M4', '1.4.18')]
sanity_check_paths = {
'files': ["bin/%s" % x for x in ["bison", "yacc"]] + ["lib/liby.a"],
'dirs': [],
}
moduleclass = 'lang'
name = 'binutils'
version = '2.32'
homepage = 'https://directory.fsf.org/project/binutils/'
description = "binutils: GNU binary utilities"
site_contacts = 'sc@fz-juelich.de'
toolchain = SYSTEM
source_urls = [GNU_SOURCE]
sources = [SOURCE_TAR_GZ]
patches = [
'binutils-2.31.1-gold-ignore-discarded-note-relocts.patch',
'binutils-2.32-readd-avx512-vmovd.patch',
'binutils-2.32_gold-include-cpp-headers.patch',
]
builddependencies = [
('flex', '2.6.4'),
('Bison', '3.6.4'),
# zlib required, but being linked in statically, so not a runtime dep
('zlib', '1.2.11'),
]
# avoid build failure when makeinfo command is not available
# see https://sourceware.org/bugzilla/show_bug.cgi?id=15345
buildopts = 'MAKEINFO=true'
installopts = buildopts
moduleclass = 'tools'
name = 'binutils'
version = '2.34'
homepage = 'https://directory.fsf.org/project/binutils/'
description = "binutils: GNU binary utilities"
site_contacts = 'sc@fz-juelich.de'
toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
source_urls = [GNU_SOURCE]
sources = [SOURCE_TAR_GZ]
patches = [
'binutils-2.31.1-gold-ignore-discarded-note-relocts.patch',
'binutils-%(version)s-readd-avx512-vmovd.patch',
]
checksums = [
'53537d334820be13eeb8acb326d01c7c81418772d626715c7ae927a7d401cab3', # binutils-2.34.tar.gz
# binutils-2.31.1-gold-ignore-discarded-note-relocts.patch
'17f22cc9136d0e81cfe8cbe310328c794a78a864e7fe7ca5827ee6678f65af32',
'45ecf7f5d198dd446d1a2e2a4d46b2747eb6fb8f2bfa18d7d42769e710e85716', # binutils-2.34-readd-avx512-vmovd.patch
]
builddependencies = [
('flex', '2.6.4'),
('Bison', '3.6.4'),
# use same binutils version that was used when building GCC toolchain, to 'bootstrap' this binutils
('binutils', version, '', True)
]
dependencies = [
# zlib is a runtime dep to avoid that it gets embedded in libbfd.so,
# see https://github.com/easybuilders/easybuild-easyblocks/issues/1350
('zlib', '1.2.11'),
]
# avoid build failure when makeinfo command is not available
# see https://sourceware.org/bugzilla/show_bug.cgi?id=15345
buildopts = 'MAKEINFO=true'
installopts = buildopts
moduleclass = 'tools'
name = 'binutils'
version = '2.34'
homepage = 'https://directory.fsf.org/project/binutils/'
description = "binutils: GNU binary utilities"
site_contacts = 'sc@fz-juelich.de'
toolchain = SYSTEM
source_urls = [GNU_SOURCE]
sources = [SOURCE_TAR_GZ]
patches = [
'binutils-2.31.1-gold-ignore-discarded-note-relocts.patch',
'binutils-2.34-readd-avx512-vmovd.patch',
]
checksums = [
'53537d334820be13eeb8acb326d01c7c81418772d626715c7ae927a7d401cab3', # binutils-2.34.tar.gz
# binutils-2.31.1-gold-ignore-discarded-note-relocts.patch
'17f22cc9136d0e81cfe8cbe310328c794a78a864e7fe7ca5827ee6678f65af32',
'45ecf7f5d198dd446d1a2e2a4d46b2747eb6fb8f2bfa18d7d42769e710e85716', # binutils-2.34-readd-avx512-vmovd.patch
]
builddependencies = [
('flex', '2.6.4'),
('Bison', '3.6.4'),
# zlib required, but being linked in statically, so not a runtime dep
('zlib', '1.2.11'),
]
# avoid build failure when makeinfo command is not available
# see https://sourceware.org/bugzilla/show_bug.cgi?id=15345
buildopts = 'MAKEINFO=true'
installopts = buildopts
moduleclass = 'tools'
name = 'CUDA'
version = '11.0.207'
local_short_version = '.'.join(version.split('.')[:2]+[version.split('.')[2][0]])
homepage = 'https://developer.nvidia.com/cuda-toolkit'
description = """CUDA (formerly Compute Unified Device Architecture) is a parallel
computing platform and programming model created by NVIDIA and implemented by the
graphics processing units (GPUs) that they produce. CUDA gives developers access
to the virtual instruction set and memory of the parallel computational elements
in CUDA GPUs.
"""
site_contacts = 'sc@fz-juelich.de'
toolchain = SYSTEM
source_urls = [
'http://developer.download.nvidia.com/compute/cuda/%(version_major_minor)s/Prod/local_installers/',
'https://developer.nvidia.com/compute/cuda/%(version_major_minor)s/prod/local_installers/',
'https://developer.nvidia.com/compute/cuda/%(version_major_minor)s/Prod2/local_installers/',
]
sources = [
'%%(namelower)s_%s_450.51.05_linux.run' % local_short_version,
]
host_compilers = [ 'g++', 'pgc++' ]
dependencies = [
('nvidia-driver', 'default', '', True),
]
installopts = '--samplespath=%(installdir)s --samples'
modluafooter = '''
add_property("arch","gpu")
'''
moduleclass = 'system'
name = 'flex'
version = '2.6.4'
homepage = 'http://flex.sourceforge.net/'
description = """Flex (Fast Lexical Analyzer) is a tool for generating scanners. A scanner,
sometimes called a tokenizer, is a program which recognizes lexical patterns in text.
"""
site_contacts = 'sc@fz-juelich.de'
toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
toolchainopts = {'pic': True}
sources = [SOURCELOWER_TAR_GZ]
source_urls = ['https://github.com/westes/flex/releases/download/v%(version)s/']
dependencies = [('Bison', '3.6.4')]
# use same binutils version that was used when building GCC toolchain
builddependencies = [('binutils', '2.34', '', True)]
parallel = 1
moduleclass = 'lang'
name = 'flex'
version = '2.6.4'
homepage = 'http://flex.sourceforge.net/'
description = """Flex (Fast Lexical Analyzer) is a tool for generating scanners. A scanner,
sometimes called a tokenizer, is a program which recognizes lexical patterns in text."""
site_contacts = 'sc@fz-juelich.de'
toolchain = SYSTEM
toolchainopts = {'pic': True}
sources = [SOURCELOWER_TAR_GZ]
source_urls = ['https://github.com/westes/flex/releases/download/v%(version)s/']
builddependencies = [
('M4', '1.4.18'),
]
dependencies = [
('Bison', '3.6.4'),
]
hidden = True
moduleclass = 'lang'
easyblock = 'Bundle'
name = 'GCC'
version = '9.3.0'
homepage = 'http://gcc.gnu.org/'
description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada,
as well as libraries for these languages (libstdc++, libgcj,...).
"""
site_contacts = 'Damian Alvarez <d.alvarez@fz-juelich.de>'
toolchain = SYSTEM
dependencies = [
('GCCcore', version),
# binutils built on top of GCCcore, which was built on top of (dummy-built) binutils
('binutils', '2.34', '', ('GCCcore', version)),
]
altroot = 'GCCcore'
altversion = 'GCCcore'
# We use a HMNS, so let's enforce a unique compiler
modluafooter = 'family("compiler")'
# Always do a recursive unload on compilers
recursive_module_unload = True
# this bundle serves as a compiler-only toolchain, so it should be marked as compiler (important for HMNS)
moduleclass = 'compiler'
easyblock = 'EB_GCC'
name = 'GCCcore'
version = '10.1.0'
local_newlib_version = '3.3.0'
homepage = 'https://gcc.gnu.org/'
description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada,
as well as libraries for these languages (libstdc++, libgcj,...)."""
site_contacts = 'Damian Alvarez <d.alvarez@fz-juelich.de>'
toolchain = SYSTEM
source_urls = [
'https://ftpmirror.gnu.org/gnu/gcc/gcc-%(version)s', # GCC auto-resolving HTTP mirror
'https://ftpmirror.gnu.org/gnu/gmp', # idem for GMP
'https://ftpmirror.gnu.org/gnu/mpfr', # idem for MPFR
'https://ftpmirror.gnu.org/gnu/mpc', # idem for MPC
'ftp://gcc.gnu.org/pub/gcc/infrastructure/', # GCC dependencies
'http://gcc.cybermirror.org/infrastructure/', # HTTP mirror for GCC dependencies
'http://isl.gforge.inria.fr/', # original HTTP source for ISL
'ftp://sourceware.org/pub/newlib/', # newlib
]
sources = [
'gcc-%(version)s.tar.gz',
'gmp-6.2.0.tar.bz2',
'mpfr-4.0.2.tar.bz2',
'mpc-1.1.0.tar.gz',
'isl-0.22.1.tar.bz2',
'newlib-%s.tar.gz' % local_newlib_version,
]
patches = [
'GCCcore-6.2.0-fix-find-isl.patch',
'GCCcore-9.3.0_gmp-c99.patch',
]
checksums = [
'954057239c89d25bc7a62bfbceb58026363ad74f079c63fdba27f95abbf60900', # gcc-10.1.0.tar.gz
'f51c99cb114deb21a60075ffb494c1a210eb9d7cb729ed042ddb7de9534451ea', # gmp-6.2.0.tar.bz2
'c05e3f02d09e0e9019384cdd58e0f19c64e6db1fd6f5ecf77b4b1c61ca253acc', # mpfr-4.0.2.tar.bz2
'6985c538143c1208dcb1ac42cedad6ff52e267b47e5f970183a3e75125b43c2e', # mpc-1.1.0.tar.gz
'1a668ef92eb181a7c021e8531a3ca89fd71aa1b3744db56f68365ab0a224c5cd', # isl-0.22.1.tar.bz2
'58dd9e3eaedf519360d92d84205c3deef0b3fc286685d1c562e245914ef72c66', # newlib-3.3.0.tar.gz
'5ad909606d17d851c6ad629b4fddb6c1621844218b8d139fed18c502a7696c68', # GCCcore-6.2.0-fix-find-isl.patch
'0e135e1cc7cec701beea9d7d17a61bab34cfd496b4b555930016b98db99f922e', # GCCcore-9.3.0_gmp-c99.patch
]
builddependencies = [
('M4', '1.4.18'),
('binutils', '2.34'),
('nvptx-tools', '20180301'),
('CUDA', '11.0.207'),
]
languages = ['c', 'c++', 'fortran']
withisl = True
# Disabled to support the nvptx target, enabled manually for x86_64 later
withlto = False
preconfigopts = 'ln -s %%(builddir)s/newlib-%s/newlib %%(builddir)s/gcc-%s && ' % (local_newlib_version, version)
configopts = [
'--target=nvptx-none --enable-as-accelerator-for=x86_64-pc-linux-gnu ' +
'--with-build-time-tools=$EBROOTNVPTXMINTOOLS/nvptx-none/bin ' +
'--disable-sjlj-exceptions --enable-newlib-io-long-long '#,
#'--enable-lto' +
#'--enable-offload-targets=nvptx-none=%(installdir)s/nvptx-none' +
#'--with-cuda-driver=$EBROOTCUDA'
]
# Make sure we replace the system cc with gcc
modaliases = {'cc': 'gcc'}
modluafooter = '''
add_property("arch","gpu")
'''
moduleclass = 'compiler'
easyblock = 'EB_GCC'
name = 'GCCcore'
version = '9.3.0'
local_newlib_version = '3.3.0'
homepage = 'https://gcc.gnu.org/'
description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada,
as well as libraries for these languages (libstdc++, libgcj,...)."""
site_contacts = 'Damian Alvarez <d.alvarez@fz-juelich.de>'
toolchain = SYSTEM
source_urls = [
'https://ftpmirror.gnu.org/gnu/gcc/gcc-%(version)s', # GCC auto-resolving HTTP mirror
'https://ftpmirror.gnu.org/gnu/gmp', # idem for GMP
'https://ftpmirror.gnu.org/gnu/mpfr', # idem for MPFR
'https://ftpmirror.gnu.org/gnu/mpc', # idem for MPC
'ftp://gcc.gnu.org/pub/gcc/infrastructure/', # GCC dependencies
'http://gcc.cybermirror.org/infrastructure/', # HTTP mirror for GCC dependencies
'http://isl.gforge.inria.fr/', # original HTTP source for ISL
'ftp://sourceware.org/pub/newlib/', # newlib
]
sources = [
'gcc-%(version)s.tar.gz',
'gmp-6.2.0.tar.bz2',
'mpfr-4.0.2.tar.bz2',
'mpc-1.1.0.tar.gz',
'isl-0.22.1.tar.bz2',
]
patches = [
'GCCcore-6.2.0-fix-find-isl.patch',
'GCCcore-%(version)s_gmp-c99.patch',
]
checksums = [
'5258a9b6afe9463c2e56b9e8355b1a4bee125ca828b8078f910303bc2ef91fa6', # gcc-9.3.0.tar.gz
'f51c99cb114deb21a60075ffb494c1a210eb9d7cb729ed042ddb7de9534451ea', # gmp-6.2.0.tar.bz2
'c05e3f02d09e0e9019384cdd58e0f19c64e6db1fd6f5ecf77b4b1c61ca253acc', # mpfr-4.0.2.tar.bz2
'6985c538143c1208dcb1ac42cedad6ff52e267b47e5f970183a3e75125b43c2e', # mpc-1.1.0.tar.gz
'1a668ef92eb181a7c021e8531a3ca89fd71aa1b3744db56f68365ab0a224c5cd', # isl-0.22.1.tar.bz2
'5ad909606d17d851c6ad629b4fddb6c1621844218b8d139fed18c502a7696c68', # GCCcore-6.2.0-fix-find-isl.patch
'0e135e1cc7cec701beea9d7d17a61bab34cfd496b4b555930016b98db99f922e', # GCCcore-9.3.0_gmp-c99.patch
]
builddependencies = [
('M4', '1.4.18'),
('binutils', '2.34'),
]
languages = ['c', 'c++', 'fortran']
withisl = True
# Make sure we replace the system cc with gcc
modaliases = {'cc': 'gcc'}
moduleclass = 'compiler'
diff -ruN gcc-9.3.0.orig/gcc/config/nvptx/nvptx.c gcc-9.3.0/gcc/config/nvptx/nvptx.c
--- gcc-9.3.0.orig/gcc/config/nvptx/nvptx.c 2020-07-15 23:11:53.663897096 +0200
+++ gcc-9.3.0/gcc/config/nvptx/nvptx.c 2020-07-15 23:14:56.885947516 +0200
@@ -5192,10 +5192,7 @@
{
fputs ("// BEGIN PREAMBLE\n", asm_out_file);
fputs ("\t.version\t3.1\n", asm_out_file);
- if (TARGET_SM35)
- fputs ("\t.target\tsm_35\n", asm_out_file);
- else
- fputs ("\t.target\tsm_30\n", asm_out_file);
+ fputs ("\t.target\tsm_35\n", asm_out_file);
fprintf (asm_out_file, "\t.address_size %d\n", GET_MODE_BITSIZE (Pmode));
fputs ("// END PREAMBLE\n", asm_out_file);
}
diff -ruN gcc-9.3.0.orig/gcc/config/nvptx/nvptx.opt gcc-9.3.0/gcc/config/nvptx/nvptx.opt
--- gcc-9.3.0.orig/gcc/config/nvptx/nvptx.opt 2020-07-15 23:11:53.663897096 +0200
+++ gcc-9.3.0/gcc/config/nvptx/nvptx.opt 2020-07-15 23:14:24.665640891 +0200
@@ -54,9 +54,6 @@
Known PTX ISA versions (for use with the -misa= option):
EnumValue
-Enum(ptx_isa) String(sm_30) Value(PTX_ISA_SM30)
-
-EnumValue
Enum(ptx_isa) String(sm_35) Value(PTX_ISA_SM35)
misa=
diff -ruN gcc-9.3.0.orig/gcc/testsuite/gcc.target/nvptx/atomic_fetch-2.c gcc-9.3.0/gcc/testsuite/gcc.target/nvptx/atomic_fetch-2.c
--- gcc-9.3.0.orig/gcc/testsuite/gcc.target/nvptx/atomic_fetch-2.c 2020-07-15 23:11:58.558791537 +0200
+++ gcc-9.3.0/gcc/testsuite/gcc.target/nvptx/atomic_fetch-2.c 2020-07-15 23:25:27.883375555 +0200
@@ -2,7 +2,7 @@
targets. */
/* { dg-do compile } */
-/* { dg-options "-O2 -misa=sm_30" } */
+/* { dg-options "-O2 -misa=sm_35" } */
int
main()
diff -ruN gcc-9.3.0.orig/libgomp/testsuite/libgomp.oacc-c-c++-common/subr.ptx gcc-9.3.0/libgomp/testsuite/libgomp.oacc-c-c++-common/subr.ptx
--- gcc-9.3.0.orig/libgomp/testsuite/libgomp.oacc-c-c++-common/subr.ptx 2020-07-15 23:11:52.778916181 +0200
+++ gcc-9.3.0/libgomp/testsuite/libgomp.oacc-c-c++-common/subr.ptx 2020-07-15 23:14:10.091955141 +0200
@@ -1,6 +1,6 @@
// BEGIN PREAMBLE
.version 3.1
- .target sm_30
+ .target sm_35
.address_size 64
// END PREAMBLE
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment