diff --git a/Custom_EasyBlocks/julia.py b/Custom_EasyBlocks/julia.py new file mode 100644 index 0000000000000000000000000000000000000000..51f485136bce56c82533fffa4dbe9caa8a977fc8 --- /dev/null +++ b/Custom_EasyBlocks/julia.py @@ -0,0 +1,236 @@ +## +# Copyright 2009-2019 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>. +## +""" +EasyBuild support for building and installing Julia packages, implemented as an easyblock + +@author: Victor Holanda (CSCS) +@author: Samuel Omlin (CSCS) +minor adjustments by Jens Henrik Goebbert (JSC) +""" +import os +import socket + +from easybuild.tools.config import build_option +from easybuild.framework.easyconfig import CUSTOM +from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.tools import systemtools + + +class EB_Julia(ConfigureMake): + """ + Install an Julia package as a separate module, or as an extension. + """ + @staticmethod + def extra_options(extra_vars=None): + extra_vars = { + 'system_name': [None, "Change julia's Project.toml pathname", CUSTOM], + 'arch_name': [None, "Change julia's Project.toml pathname", CUSTOM], + } + return ConfigureMake.extra_options(extra_vars) + + + def get_environment_folder(self): + env_path = '' + + systemname = 'default' + if self.cfg['system_name']: + systemname = self.cfg['system_name'] + + if self.cfg['arch_name']: + env_path = '-'.join([systemname, self.cfg['arch_name']]) + return env_path + + arch = systemtools.get_cpu_architecture() + cpu_family = systemtools.get_cpu_family() + env_path = '-'.join([systemname, cpu_family, arch]) + return env_path + + def get_user_depot_path(self): + user_depot_path = '' + + arch = systemtools.get_cpu_architecture() + cpu_family = systemtools.get_cpu_family() + user_depot_path = os.path.join('~', '.julia', self.version, self.get_environment_folder()) + return user_depot_path + + def __init__(self, *args, **kwargs): + super(EB_Julia, self).__init__(*args, **kwargs) + + self.user_depot = self.get_user_depot_path() + local_share_depot = os.path.join(self.installdir, 'local', 'share', 'julia') + share_depot = os.path.join(self.installdir, 'share', 'julia') + self.std_depots = ':'.join([local_share_depot, share_depot]) + self.julia_depot_path = ':'.join([self.user_depot, self.std_depots]) + self.admin_depots = os.path.join(self.installdir, 'extensions') + + self.julia_project = os.path.join(self.user_depot, "environments", '-'.join([self.version, self.get_environment_folder()])) + + self.user_load_path = '@:@#.#.#-%s' % self.get_environment_folder() + self.std_load_paths = '@stdlib' + self.julia_load_path = ':'.join([self.user_load_path, self.std_load_paths]) + self.admin_load_path = os.path.join(self.admin_depots, "environments", '-'.join([self.version, self.get_environment_folder()])) + + def sanity_check_step(self): + """Custom sanity check for Julia.""" + + custom_paths = { + 'files': [os.path.join('bin', 'julia'), 'LICENSE.md'], + 'dirs': ['bin', 'include', 'lib', 'share'], + } + custom_commands = [ + "julia --version", + "julia --eval '1+2'", + ] + + super(EB_Julia, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + + def install_step(self, *args, **kwargs): + """Install procedure for Julia""" + + super(EB_Julia, self).install_step(*args, **kwargs) + txt = """ +## Read EB environment variables + +if haskey(ENV, "EBJULIA_ADMIN_LOAD_PATH") + ADMIN_LOAD_PATH = split(ENV["EBJULIA_ADMIN_LOAD_PATH"],':') +else + ADMIN_LOAD_PATH = [] +end + +if haskey(ENV, "EBJULIA_STD_LOAD_PATH") + STD_LOAD_PATH = split(ENV["EBJULIA_STD_LOAD_PATH"],':') +else + STD_LOAD_PATH = [] +end + +if haskey(ENV, "EBJULIA_ADMIN_DEPOT_PATH") + ADMIN_DEPOT_PATH = split(ENV["EBJULIA_ADMIN_DEPOT_PATH"],':') +else + ADMIN_DEPOT_PATH = [] +end + +if haskey(ENV, "EBJULIA_STD_DEPOT_PATH") + STD_DEPOT_PATH = split(ENV["EBJULIA_STD_DEPOT_PATH"],':') +else + STD_DEPOT_PATH = [] +end + + +## Inject the admin paths, except if paths empty (or only "@" for LOAD_PATH) or all entries in std path. +if !( isempty(LOAD_PATH) || isempty(DEPOT_PATH) || (length(LOAD_PATH)==1 && LOAD_PATH[1]=="@") || + all([entry in STD_LOAD_PATH for entry in LOAD_PATH]) || all([entry in STD_DEPOT_PATH for entry in DEPOT_PATH]) ) + + ## Inject the admin load path into the LOAD_PATH + + # Empty the LOAD_PATH, separating load path into user and std load path. + user_load_path = [] + std_load_path = [] + while !isempty(LOAD_PATH) + entry = popfirst!(LOAD_PATH) + if entry in STD_LOAD_PATH + push!(std_load_path, entry) + else + push!(user_load_path, entry) + end + end + + # Add user load path to LOAD_PATH + while !isempty(user_load_path) + entry = popfirst!(user_load_path) + push!(LOAD_PATH, entry) + end + + # Add admin load path to LOAD_PATH + while !isempty(ADMIN_LOAD_PATH) + entry = popfirst!(ADMIN_LOAD_PATH) + push!(LOAD_PATH, entry) + end + + # Add std load path to LOAD_PATH + while !isempty(std_load_path) + entry = popfirst!(std_load_path) + push!(LOAD_PATH, entry) + end + + + ## Inject the admin depot path into the DEPOT_PATH + + # Empty the DEPOT_PATH, separating depots into user and std depots. + user_depot_path = [] + std_depot_path = [] + while !isempty(DEPOT_PATH) + depot = popfirst!(DEPOT_PATH) + if depot in STD_DEPOT_PATH + push!(std_depot_path, depot) + else + push!(user_depot_path, depot) + end + end + + # Add user depots to DEPOT_PATH + while !isempty(user_depot_path) + depot = popfirst!(user_depot_path) + push!(DEPOT_PATH, depot) + end + + # Add admin depots to DEPOT_PATH + while !isempty(ADMIN_DEPOT_PATH) + depot = popfirst!(ADMIN_DEPOT_PATH) + push!(DEPOT_PATH, depot) + end + + # Add std depots to DEPOT_PATH + while !isempty(std_depot_path) + depot = popfirst!(std_depot_path) + push!(DEPOT_PATH, depot) + end + +end + + """ + with open(os.path.join(self.installdir, 'etc', 'julia', 'startup.jl'), 'w') as startup_file: + startup_file.write(txt) + startup_file.close() + + def make_module_extra(self, *args, **kwargs): + txt = super(EB_Julia, self).make_module_extra(*args, **kwargs) + + txt += self.module_generator.set_environment('JULIA_INSTALL_FOLDER', self.installdir) + + txt += self.module_generator.set_environment('JULIA_PROJECT', self.julia_project) + txt += self.module_generator.set_environment('JULIA_DEPOT_PATH', self.julia_depot_path) + txt += self.module_generator.set_environment('EBJULIA_USER_DEPOT_PATH', self.user_depot) + txt += self.module_generator.set_environment('EBJULIA_ADMIN_DEPOT_PATH', self.admin_depots) + txt += self.module_generator.set_environment('EBJULIA_STD_DEPOT_PATH', self.std_depots) + + + txt += self.module_generator.set_environment('JULIA_LOAD_PATH', self.julia_load_path) + txt += self.module_generator.set_environment('EBJULIA_USER_LOAD_PATH', self.user_load_path) + txt += self.module_generator.set_environment('EBJULIA_ADMIN_LOAD_PATH', self.admin_load_path) + txt += self.module_generator.set_environment('EBJULIA_STD_LOAD_PATH', self.std_load_paths) + + txt += self.module_generator.set_environment('EBJULIA_ENV_NAME', '-'.join([self.version, self.get_environment_folder()])) + + return txt diff --git a/Custom_EasyBlocks/juliabundle.py b/Custom_EasyBlocks/juliabundle.py new file mode 100644 index 0000000000000000000000000000000000000000..6a6dc4352ec0a52e5a6139d1315768c1c12a6dbd --- /dev/null +++ b/Custom_EasyBlocks/juliabundle.py @@ -0,0 +1,116 @@ +## +# Copyright 2009-2019 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>. +## +""" +EasyBuild support for building and installing Julia packages, implemented as an easyblock + +@author: Victor Holanda (CSCS) +@author: Samuel Omlin (CSCS) +minor adjustments by Jens Henrik Goebbert (JSC) +""" +import os +import socket + +from easybuild.easyblocks.generic.bundle import Bundle +from easybuild.tools.config import build_option +from easybuild.tools import systemtools +#from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir +from .juliapackage import JuliaPackage + + +class JuliaBundle(Bundle): + """ + Install an Julia package as a separate module, or as an extension. + """ + + @staticmethod + def extra_options(extra_vars=None): + """Easyconfig parameters specific to bundles of Python packages.""" + #50 extra_vars = { + #51 'arch_name': [None, "Change julia's Project.toml pathname", CUSTOM], + #52 } + if extra_vars is None: + extra_vars = {} + # combine custom easyconfig parameters of Bundle & JuliaPackage + extra_vars = Bundle.extra_options(extra_vars) + return JuliaPackage.extra_options(extra_vars) + + def get_environment_folder(self): + env_path = '' + + systemname = 'default' + if self.cfg['system_name']: + systemname = self.cfg['system_name'] + + if self.cfg['arch_name']: + env_path = '-'.join([systemname, self.cfg['arch_name']]) + return env_path + + arch = systemtools.get_cpu_architecture() + cpu_family = systemtools.get_cpu_family() + env_path = '-'.join([systemname, cpu_family, arch]) + return env_path + + def __init__(self, *args, **kwargs): + super(JuliaBundle, self).__init__(*args, **kwargs) + self.cfg['exts_defaultclass'] = 'JuliaPackage' + + # need to disable templating to ensure that actual value for exts_default_options is updated... + prev_enable_templating = self.cfg.enable_templating + self.cfg.enable_templating = False + + # set default options for extensions according to relevant top-level easyconfig parameters + julpkg_keys = JuliaPackage.extra_options().keys() + for key in julpkg_keys: + if key not in self.cfg['exts_default_options']: + self.cfg['exts_default_options'][key] = self.cfg[key] + + self.cfg['exts_default_options']['download_dep_fail'] = True + self.log.info("Detection of downloaded extension dependencies is enabled") + + self.cfg.enable_templating = prev_enable_templating + + self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + + self.extensions_depot = 'extensions' + + self.admin_load_path = os.path.join(self.extensions_depot, "environments", '-'.join([self.version, self.get_environment_folder()])) + + def sanity_check_step(self): + """Custom sanity check for Julia.""" + + custom_paths = { + 'files': [], + 'dirs': ['extensions'], + } + super(JuliaBundle, self).sanity_check_step(custom_paths=custom_paths) + + def make_module_extra(self, *args, **kwargs): + txt = super(JuliaBundle, self).make_module_extra(*args, **kwargs) + + txt += self.module_generator.prepend_paths('EBJULIA_ADMIN_DEPOT_PATH', self.extensions_depot) + + txt += self.module_generator.prepend_paths('EBJULIA_ADMIN_LOAD_PATH', self.admin_load_path) + + return txt diff --git a/Custom_EasyBlocks/juliapackage.py b/Custom_EasyBlocks/juliapackage.py new file mode 100644 index 0000000000000000000000000000000000000000..58cf4cd4b285828e0e936c14f9c31a1806565a50 --- /dev/null +++ b/Custom_EasyBlocks/juliapackage.py @@ -0,0 +1,153 @@ +## +# Copyright 2009-2019 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>. +## +""" +EasyBuild support for building and installing Julia packages, implemented as an easyblock + +@author: Victor Holanda (CSCS) +@author: Samuel Omlin (CSCS) +minor adjustments by Jens Henrik Goebbert (JSC) +""" +import os +import sys + +import easybuild.tools.toolchain as toolchain + +from easybuild.framework.easyconfig import CUSTOM +from easybuild.framework.extensioneasyblock import ExtensionEasyBlock +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.run import run_cmd, parse_log_for_error + + +class JuliaPackage(ExtensionEasyBlock): + """ + Install an Julia package as a separate module, or as an extension. + """ + + @staticmethod + def extra_options(extra_vars=None): + if extra_vars is None: + extra_vars = {} + + extra_vars.update({ + 'system_name': [None, "Change julia's Project.toml pathname", CUSTOM], + 'arch_name': [None, "Change julia's Project.toml pathname", CUSTOM], + 'packagespec': [None, "Overwrite install options for Pkg.add(PackageSpec(<packagespec>))", CUSTOM], + 'mpiexec': [None, "Set the mpiexec command", CUSTOM], + 'mpiexec_args': [None, "Set the mpiexec command args", CUSTOM], + 'mpi_path': [None, "Set the MPI installation path", CUSTOM], + 'mpicc': [None, "Set mpicc command", "mpicc"], + }) + return ExtensionEasyBlock.extra_options(extra_vars=extra_vars) + + def __init__(self, *args, **kwargs): + super(JuliaPackage, self).__init__(*args, **kwargs) + self.package_name = self.name + names = self.package_name.split('.') + if len(names) > 1: + self.package_name = ''.join(names[:-1]) + + julia_env_name = os.getenv('EBJULIA_ENV_NAME', '') + self.depot = os.path.join(self.installdir, 'extensions') + self.projectdir = os.path.join(self.depot, 'environments', julia_env_name) + self.log.info("Depot for package installations: %s" % self.depot) + + def patch_step(self, beginpath=None): + pass + + def fetch_sources(self, sources=None, checksums=None): + pass + + def extract_step(self): + """Source should not be extracted.""" + pass + + def configure_step(self): + """No configuration for installing Julia packages.""" + pass + + def build_step(self): + """No separate build step for Julia packages.""" + pass + + def make_julia_cmd(self, remove=False): + """Create a command to run in julia to install an julia package.""" + + if self.cfg['packagespec']: + package_spec = self.cfg['packagespec'] + else: + package_spec = "name=\"%s\", version=\"%s\"" % (self.package_name, self.version) + + pre_cmd = '%s unset EBJULIA_USER_DEPOT_PATH && unset EBJULIA_ADMIN_DEPOT_PATH && export JULIA_DEPOT_PATH=%s && export JULIA_PROJECT=%s' % (self.cfg['preinstallopts'], self.depot, self.projectdir) + + if self.cfg['mpi_path']: + pre_cmd += ' && export JULIA_MPI_BINARY=system' + pre_cmd += ' && export JULIA_MPI_PATH="%s"' % self.cfg['mpi_path'] + + if self.cfg['mpiexec']: + pre_cmd += ' && export JULIA_MPIEXEC="%s"' % self.cfg['mpiexec'] + + if self.cfg['mpiexec_args']: + pre_cmd += ' && export JULIA_MPIEXEC_ARGS="%s"' % self.cfg['mpiexec_args'] + + if self.cfg['mpicc']: + pre_cmd += ' && export JULIA_MPICC="%s"' % self.cfg['mpicc'] + + if self.cfg['arch_name'] == 'gpu': + pre_cmd += ' && export JULIA_CUDA_USE_BINARYBUILDER=false' + + if remove: + cmd = ' && '.join([pre_cmd, "julia --eval 'using Pkg; Pkg.rm(PackageSpec(%s))'" % package_spec]) + else: + cmd = ' && '.join([pre_cmd, "julia --eval 'using Pkg; Pkg.add(PackageSpec(%s))'" % package_spec]) + + return cmd + + def install_step(self): + """Install procedure for Julia packages.""" + + cmd = self.make_julia_cmd(remove=False) + cmdttdouterr, _ = run_cmd(cmd, log_all=True, simple=False, regexp=False) + + cmderrors = parse_log_for_error(cmdttdouterr, regExp="^ERROR:") + if cmderrors: + cmd = self.make_julia_cmd(remove=True) + run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=sys.stdin, regexp=False) + raise EasyBuildError("Errors detected during installation of Julia package %s!", self.name) + + self.log.info("Julia package %s installed succesfully" % self.name) + + def run(self): + """Install Julia package as an extension.""" + self.install_step() + + def sanity_check_step(self, *args, **kwargs): + """ + Custom sanity check for Julia packages + """ + #NOTE: we don't use Pkg.status with arguments as only supported for Julia >=v1.1 + cmd = "unset EBJULIA_USER_DEPOT_PATH && unset EBJULIA_ADMIN_DEPOT_PATH && export JULIA_DEPOT_PATH=%s && export JULIA_PROJECT=%s && julia --eval 'using Pkg; Pkg.status()'" % (self.depot, self.projectdir) + cmdttdouterr, _ = run_cmd(cmd, log_all=True, simple=False, regexp=False) + self.log.error("Julia package %s sanity returned %s" % (self.name, cmdttdouterr)) + return len(parse_log_for_error(cmdttdouterr, regExp="%s\s+v%s" % (self.package_name, self.version))) != 0 diff --git a/Golden_Repo/j/Julia/Julia-1.6.4-gcccoremkl-11.2.0-2021.4.0.eb b/Golden_Repo/j/Julia/Julia-1.6.4-gcccoremkl-11.2.0-2021.4.0.eb index 7ed1a29fcb7829958faafeadca88b6ec77c662a1..fea2d81d798100fc7f77dab782d218ee589c232b 100644 --- a/Golden_Repo/j/Julia/Julia-1.6.4-gcccoremkl-11.2.0-2021.4.0.eb +++ b/Golden_Repo/j/Julia/Julia-1.6.4-gcccoremkl-11.2.0-2021.4.0.eb @@ -7,7 +7,7 @@ Julia programs compile to efficient native code for multiple platforms via LLVM """ -toolchain = {'name': 'gcccoremkl', 'version': '11.2.0-2021b.4.0'} +toolchain = {'name': 'gcccoremkl', 'version': '11.2.0-2021.4.0'} toolchainopts = {'pic': True, 'verbose': True} # , 'usempi': True} source_urls = ['https://github.com/JuliaLang/julia/releases/download/v%(version)s/'] @@ -15,7 +15,7 @@ sources = ['julia-%(version)s-full.tar.gz'] builddependencies = [ ('binutils', '2.37'), - ('git', '2.28.0'), + ('git', '2.33.1', '-nodocs'), ('CMake', '3.21.1', '', SYSTEM), ] @@ -23,7 +23,7 @@ dependencies = [ ('Python', '3.9.6'), ('GMP', '6.2.1'), ('CUDA', '11.5', '', SYSTEM), - ('SciPy-Stack', '2021b', '-Python-%(pyver)s'), + ('SciPy-Stack', '2021b'), ('OpenGL', '2021b'), ('OpenSSL', '1.1', '', True), ] diff --git a/Golden_Repo/j/Julia/Julia-1.6.4-gomkl-2021b.eb b/Golden_Repo/j/Julia/Julia-1.6.4-gomkl-2021b.eb index 8de5aa9732be4b251afab336150f2bc9b99a4efb..1bf2906a489c34b092401b27894af35831405c83 100644 --- a/Golden_Repo/j/Julia/Julia-1.6.4-gomkl-2021b.eb +++ b/Golden_Repo/j/Julia/Julia-1.6.4-gomkl-2021b.eb @@ -14,7 +14,7 @@ sources = ['julia-%(version)s-full.tar.gz'] builddependencies = [ ('binutils', '2.37'), - ('git', '2.28.0'), + ('git', '2.33.1', '-nodocs'), ('CMake', '3.21.1', '', SYSTEM), ] @@ -22,7 +22,7 @@ dependencies = [ ('Python', '3.9.6'), ('GMP', '6.2.1'), ('CUDA', '11.5', '', SYSTEM), - ('SciPy-Stack', '2022', '-Python-%(pyver)s', ('gcccoremkl', '10.3.0-2021.2.0')), + ('SciPy-Stack', '2021b', '', ('gcccoremkl', '11.2.0-2021.4.0')), ('OpenGL', '2021b'), ('OpenSSL', '1.1', '', True), ]