Skip to content
Snippets Groups Projects
Commit 8c72373d authored by Jens Henrik Göbbert's avatar Jens Henrik Göbbert
Browse files

major update of our Julia-Installation

parent cbc29b49
Branches
No related tags found
No related merge requests found
##
# Copyright 2009-2019 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for building and installing Julia packages, implemented as an easyblock
@author: Victor Holanda (CSCS)
@author: Samuel Omlin (CSCS)
minor adjustments by Jens Henrik Goebbert (JSC)
"""
import os
import socket
from easybuild.tools.config import build_option
from easybuild.framework.easyconfig import CUSTOM
from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.tools import systemtools
class EB_Julia(ConfigureMake):
"""
Install an Julia package as a separate module, or as an extension.
"""
@staticmethod
def extra_options(extra_vars=None):
extra_vars = {
'system_name': [None, "Change julia's Project.toml pathname", CUSTOM],
'arch_name': [None, "Change julia's Project.toml pathname", CUSTOM],
}
return ConfigureMake.extra_options(extra_vars)
def get_environment_folder(self):
env_path = ''
systemname = 'default'
if self.cfg['system_name']:
systemname = self.cfg['system_name']
if self.cfg['arch_name']:
env_path = '-'.join([systemname, self.cfg['arch_name']])
return env_path
arch = systemtools.get_cpu_architecture()
cpu_family = systemtools.get_cpu_family()
env_path = '-'.join([systemname, cpu_family, arch])
return env_path
def get_user_depot_path(self):
user_depot_path = ''
arch = systemtools.get_cpu_architecture()
cpu_family = systemtools.get_cpu_family()
user_depot_path = os.path.join('~', '.julia', self.version, self.get_environment_folder())
return user_depot_path
def __init__(self, *args, **kwargs):
super(EB_Julia, self).__init__(*args, **kwargs)
self.user_depot = self.get_user_depot_path()
local_share_depot = os.path.join(self.installdir, 'local', 'share', 'julia')
share_depot = os.path.join(self.installdir, 'share', 'julia')
self.std_depots = ':'.join([local_share_depot, share_depot])
self.julia_depot_path = ':'.join([self.user_depot, self.std_depots])
self.admin_depots = os.path.join(self.installdir, 'extensions')
self.julia_project = os.path.join(self.user_depot, "environments", '-'.join([self.version, self.get_environment_folder()]))
self.user_load_path = '@:@#.#.#-%s' % self.get_environment_folder()
self.std_load_paths = '@stdlib'
self.julia_load_path = ':'.join([self.user_load_path, self.std_load_paths])
self.admin_load_path = os.path.join(self.admin_depots, "environments", '-'.join([self.version, self.get_environment_folder()]))
def sanity_check_step(self):
"""Custom sanity check for Julia."""
custom_paths = {
'files': [os.path.join('bin', 'julia'), 'LICENSE.md'],
'dirs': ['bin', 'include', 'lib', 'share'],
}
custom_commands = [
"julia --version",
"julia --eval '1+2'",
]
super(EB_Julia, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
def install_step(self, *args, **kwargs):
"""Install procedure for Julia"""
super(EB_Julia, self).install_step(*args, **kwargs)
txt = """
## Read EB environment variables
if haskey(ENV, "EBJULIA_ADMIN_LOAD_PATH")
ADMIN_LOAD_PATH = split(ENV["EBJULIA_ADMIN_LOAD_PATH"],':')
else
ADMIN_LOAD_PATH = []
end
if haskey(ENV, "EBJULIA_STD_LOAD_PATH")
STD_LOAD_PATH = split(ENV["EBJULIA_STD_LOAD_PATH"],':')
else
STD_LOAD_PATH = []
end
if haskey(ENV, "EBJULIA_ADMIN_DEPOT_PATH")
ADMIN_DEPOT_PATH = split(ENV["EBJULIA_ADMIN_DEPOT_PATH"],':')
else
ADMIN_DEPOT_PATH = []
end
if haskey(ENV, "EBJULIA_STD_DEPOT_PATH")
STD_DEPOT_PATH = split(ENV["EBJULIA_STD_DEPOT_PATH"],':')
else
STD_DEPOT_PATH = []
end
## Inject the admin paths, except if paths empty (or only "@" for LOAD_PATH) or all entries in std path.
if !( isempty(LOAD_PATH) || isempty(DEPOT_PATH) || (length(LOAD_PATH)==1 && LOAD_PATH[1]=="@") ||
all([entry in STD_LOAD_PATH for entry in LOAD_PATH]) || all([entry in STD_DEPOT_PATH for entry in DEPOT_PATH]) )
## Inject the admin load path into the LOAD_PATH
# Empty the LOAD_PATH, separating load path into user and std load path.
user_load_path = []
std_load_path = []
while !isempty(LOAD_PATH)
entry = popfirst!(LOAD_PATH)
if entry in STD_LOAD_PATH
push!(std_load_path, entry)
else
push!(user_load_path, entry)
end
end
# Add user load path to LOAD_PATH
while !isempty(user_load_path)
entry = popfirst!(user_load_path)
push!(LOAD_PATH, entry)
end
# Add admin load path to LOAD_PATH
while !isempty(ADMIN_LOAD_PATH)
entry = popfirst!(ADMIN_LOAD_PATH)
push!(LOAD_PATH, entry)
end
# Add std load path to LOAD_PATH
while !isempty(std_load_path)
entry = popfirst!(std_load_path)
push!(LOAD_PATH, entry)
end
## Inject the admin depot path into the DEPOT_PATH
# Empty the DEPOT_PATH, separating depots into user and std depots.
user_depot_path = []
std_depot_path = []
while !isempty(DEPOT_PATH)
depot = popfirst!(DEPOT_PATH)
if depot in STD_DEPOT_PATH
push!(std_depot_path, depot)
else
push!(user_depot_path, depot)
end
end
# Add user depots to DEPOT_PATH
while !isempty(user_depot_path)
depot = popfirst!(user_depot_path)
push!(DEPOT_PATH, depot)
end
# Add admin depots to DEPOT_PATH
while !isempty(ADMIN_DEPOT_PATH)
depot = popfirst!(ADMIN_DEPOT_PATH)
push!(DEPOT_PATH, depot)
end
# Add std depots to DEPOT_PATH
while !isempty(std_depot_path)
depot = popfirst!(std_depot_path)
push!(DEPOT_PATH, depot)
end
end
"""
with open(os.path.join(self.installdir, 'etc', 'julia', 'startup.jl'), 'w') as startup_file:
startup_file.write(txt)
startup_file.close()
def make_module_extra(self, *args, **kwargs):
txt = super(EB_Julia, self).make_module_extra(*args, **kwargs)
txt += self.module_generator.set_environment('JULIA_INSTALL_FOLDER', self.installdir)
txt += self.module_generator.set_environment('JULIA_PROJECT', self.julia_project)
txt += self.module_generator.set_environment('JULIA_DEPOT_PATH', self.julia_depot_path)
txt += self.module_generator.set_environment('EBJULIA_USER_DEPOT_PATH', self.user_depot)
txt += self.module_generator.set_environment('EBJULIA_ADMIN_DEPOT_PATH', self.admin_depots)
txt += self.module_generator.set_environment('EBJULIA_STD_DEPOT_PATH', self.std_depots)
txt += self.module_generator.set_environment('JULIA_LOAD_PATH', self.julia_load_path)
txt += self.module_generator.set_environment('EBJULIA_USER_LOAD_PATH', self.user_load_path)
txt += self.module_generator.set_environment('EBJULIA_ADMIN_LOAD_PATH', self.admin_load_path)
txt += self.module_generator.set_environment('EBJULIA_STD_LOAD_PATH', self.std_load_paths)
txt += self.module_generator.set_environment('EBJULIA_ENV_NAME', '-'.join([self.version, self.get_environment_folder()]))
return txt
##
# Copyright 2009-2019 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for building and installing Julia packages, implemented as an easyblock
@author: Victor Holanda (CSCS)
@author: Samuel Omlin (CSCS)
minor adjustments by Jens Henrik Goebbert (JSC)
"""
import os
import socket
from easybuild.easyblocks.generic.bundle import Bundle
from easybuild.tools.config import build_option
from easybuild.tools import systemtools
#from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir
from .juliapackage import JuliaPackage
class JuliaBundle(Bundle):
"""
Install an Julia package as a separate module, or as an extension.
"""
@staticmethod
def extra_options(extra_vars=None):
"""Easyconfig parameters specific to bundles of Python packages."""
#50 extra_vars = {
#51 'arch_name': [None, "Change julia's Project.toml pathname", CUSTOM],
#52 }
if extra_vars is None:
extra_vars = {}
# combine custom easyconfig parameters of Bundle & JuliaPackage
extra_vars = Bundle.extra_options(extra_vars)
return JuliaPackage.extra_options(extra_vars)
def get_environment_folder(self):
env_path = ''
systemname = 'default'
if self.cfg['system_name']:
systemname = self.cfg['system_name']
if self.cfg['arch_name']:
env_path = '-'.join([systemname, self.cfg['arch_name']])
return env_path
arch = systemtools.get_cpu_architecture()
cpu_family = systemtools.get_cpu_family()
env_path = '-'.join([systemname, cpu_family, arch])
return env_path
def __init__(self, *args, **kwargs):
super(JuliaBundle, self).__init__(*args, **kwargs)
self.cfg['exts_defaultclass'] = 'JuliaPackage'
# need to disable templating to ensure that actual value for exts_default_options is updated...
prev_enable_templating = self.cfg.enable_templating
self.cfg.enable_templating = False
# set default options for extensions according to relevant top-level easyconfig parameters
julpkg_keys = JuliaPackage.extra_options().keys()
for key in julpkg_keys:
if key not in self.cfg['exts_default_options']:
self.cfg['exts_default_options'][key] = self.cfg[key]
self.cfg['exts_default_options']['download_dep_fail'] = True
self.log.info("Detection of downloaded extension dependencies is enabled")
self.cfg.enable_templating = prev_enable_templating
self.log.info("exts_default_options: %s", self.cfg['exts_default_options'])
self.extensions_depot = 'extensions'
self.admin_load_path = os.path.join(self.extensions_depot, "environments", '-'.join([self.version, self.get_environment_folder()]))
def sanity_check_step(self):
"""Custom sanity check for Julia."""
custom_paths = {
'files': [],
'dirs': ['extensions'],
}
super(JuliaBundle, self).sanity_check_step(custom_paths=custom_paths)
def make_module_extra(self, *args, **kwargs):
txt = super(JuliaBundle, self).make_module_extra(*args, **kwargs)
txt += self.module_generator.prepend_paths('EBJULIA_ADMIN_DEPOT_PATH', self.extensions_depot)
txt += self.module_generator.prepend_paths('EBJULIA_ADMIN_LOAD_PATH', self.admin_load_path)
return txt
##
# Copyright 2009-2019 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for building and installing Julia packages, implemented as an easyblock
@author: Victor Holanda (CSCS)
@author: Samuel Omlin (CSCS)
minor adjustments by Jens Henrik Goebbert (JSC)
"""
import os
import sys
import easybuild.tools.toolchain as toolchain
from easybuild.framework.easyconfig import CUSTOM
from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.run import run_cmd, parse_log_for_error
class JuliaPackage(ExtensionEasyBlock):
"""
Install an Julia package as a separate module, or as an extension.
"""
@staticmethod
def extra_options(extra_vars=None):
if extra_vars is None:
extra_vars = {}
extra_vars.update({
'system_name': [None, "Change julia's Project.toml pathname", CUSTOM],
'arch_name': [None, "Change julia's Project.toml pathname", CUSTOM],
'packagespec': [None, "Overwrite install options for Pkg.add(PackageSpec(<packagespec>))", CUSTOM],
'mpiexec': [None, "Set the mpiexec command", CUSTOM],
'mpiexec_args': [None, "Set the mpiexec command args", CUSTOM],
'mpi_path': [None, "Set the MPI installation path", CUSTOM],
'mpicc': [None, "Set mpicc command", "mpicc"],
})
return ExtensionEasyBlock.extra_options(extra_vars=extra_vars)
def __init__(self, *args, **kwargs):
super(JuliaPackage, self).__init__(*args, **kwargs)
self.package_name = self.name
names = self.package_name.split('.')
if len(names) > 1:
self.package_name = ''.join(names[:-1])
julia_env_name = os.getenv('EBJULIA_ENV_NAME', '')
self.depot = os.path.join(self.installdir, 'extensions')
self.projectdir = os.path.join(self.depot, 'environments', julia_env_name)
self.log.info("Depot for package installations: %s" % self.depot)
def patch_step(self, beginpath=None):
pass
def fetch_sources(self, sources=None, checksums=None):
pass
def extract_step(self):
"""Source should not be extracted."""
pass
def configure_step(self):
"""No configuration for installing Julia packages."""
pass
def build_step(self):
"""No separate build step for Julia packages."""
pass
def make_julia_cmd(self, remove=False):
"""Create a command to run in julia to install an julia package."""
if self.cfg['packagespec']:
package_spec = self.cfg['packagespec']
else:
package_spec = "name=\"%s\", version=\"%s\"" % (self.package_name, self.version)
pre_cmd = '%s unset EBJULIA_USER_DEPOT_PATH && unset EBJULIA_ADMIN_DEPOT_PATH && export JULIA_DEPOT_PATH=%s && export JULIA_PROJECT=%s' % (self.cfg['preinstallopts'], self.depot, self.projectdir)
if self.cfg['mpi_path']:
pre_cmd += ' && export JULIA_MPI_BINARY=system'
pre_cmd += ' && export JULIA_MPI_PATH="%s"' % self.cfg['mpi_path']
if self.cfg['mpiexec']:
pre_cmd += ' && export JULIA_MPIEXEC="%s"' % self.cfg['mpiexec']
if self.cfg['mpiexec_args']:
pre_cmd += ' && export JULIA_MPIEXEC_ARGS="%s"' % self.cfg['mpiexec_args']
if self.cfg['mpicc']:
pre_cmd += ' && export JULIA_MPICC="%s"' % self.cfg['mpicc']
if self.cfg['arch_name'] == 'gpu':
pre_cmd += ' && export JULIA_CUDA_USE_BINARYBUILDER=false'
if remove:
cmd = ' && '.join([pre_cmd, "julia --eval 'using Pkg; Pkg.rm(PackageSpec(%s))'" % package_spec])
else:
cmd = ' && '.join([pre_cmd, "julia --eval 'using Pkg; Pkg.add(PackageSpec(%s))'" % package_spec])
return cmd
def install_step(self):
"""Install procedure for Julia packages."""
cmd = self.make_julia_cmd(remove=False)
cmdttdouterr, _ = run_cmd(cmd, log_all=True, simple=False, regexp=False)
cmderrors = parse_log_for_error(cmdttdouterr, regExp="^ERROR:")
if cmderrors:
cmd = self.make_julia_cmd(remove=True)
run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=sys.stdin, regexp=False)
raise EasyBuildError("Errors detected during installation of Julia package %s!", self.name)
self.log.info("Julia package %s installed succesfully" % self.name)
def run(self):
"""Install Julia package as an extension."""
self.install_step()
def sanity_check_step(self, *args, **kwargs):
"""
Custom sanity check for Julia packages
"""
#NOTE: we don't use Pkg.status with arguments as only supported for Julia >=v1.1
cmd = "unset EBJULIA_USER_DEPOT_PATH && unset EBJULIA_ADMIN_DEPOT_PATH && export JULIA_DEPOT_PATH=%s && export JULIA_PROJECT=%s && julia --eval 'using Pkg; Pkg.status()'" % (self.depot, self.projectdir)
cmdttdouterr, _ = run_cmd(cmd, log_all=True, simple=False, regexp=False)
self.log.error("Julia package %s sanity returned %s" % (self.name, cmdttdouterr))
return len(parse_log_for_error(cmdttdouterr, regExp="%s\s+v%s" % (self.package_name, self.version))) != 0
easyblock = 'JuliaBundle'
name = 'Julia.CUDA'
version = '1.6.1'
homepage = 'https://juliagpu.gitlab.io/CUDA.jl'
description = """The CUDA.jl package is the main entrypoint for for programming NVIDIA GPUs using CUDA.
The package makes it possible to do so at various abstraction levels,
from easy-to-use arrays down to hand-written kernels using low-level CUDA APIs.
"""
site_contacts = 'j.goebbert@fz-juelich.de'
toolchain = {'name': 'gcccoremkl', 'version': '9.3.0-2020.2.254'}
toolchainopts = {'pic': True}
builddependencies = [
('binutils', '2.34'),
]
dependencies = [
('Julia', '1.6.1'),
('CUDA', '11.0', '', SYSTEM),
]
arch_name = 'gpu'
exts_defaultclass = 'JuliaPackage'
exts_list = [
('CUDA.jl', '3.1.0', {
'source_tmpl': 'v3.1.0.tar.gz',
'source_urls': ['https://github.com/JuliaGPU/CUDA.jl/archive/']
}),
]
modextravars = {
'JULIA_CUDA_USE_BINARYBUILDER': 'false',
}
moduleclass = 'tools'
easyblock = 'ConfigureMake' # For using $SYSTEMNAME to determine compute capability. The local prefix is to appease the checker
import os as local_os
name = 'Julia' name = 'Julia'
version = '1.6.1' version = '1.6.1'
...@@ -10,7 +12,7 @@ Julia programs compile to efficient native code for multiple platforms via LLVM ...@@ -10,7 +12,7 @@ Julia programs compile to efficient native code for multiple platforms via LLVM
site_contacts = 'j.goebbert@fz-juelich.de' site_contacts = 'j.goebbert@fz-juelich.de'
toolchain = {'name': 'gcccoremkl', 'version': '9.3.0-2020.2.254'} toolchain = {'name': 'gcccoremkl', 'version': '9.3.0-2020.2.254'}
toolchainopts = {'pic': True} toolchainopts = {'pic': True, 'verbose': True} # , 'usempi': True}
source_urls = ['https://github.com/JuliaLang/julia/releases/download/v%(version)s/'] source_urls = ['https://github.com/JuliaLang/julia/releases/download/v%(version)s/']
sources = ['julia-%(version)s-full.tar.gz'] sources = ['julia-%(version)s-full.tar.gz']
...@@ -23,127 +25,150 @@ builddependencies = [ ...@@ -23,127 +25,150 @@ builddependencies = [
dependencies = [ dependencies = [
('Python', '3.8.5'), ('Python', '3.8.5'),
('GMP', '6.2.0'),
('SciPy-Stack', '2020', '-Python-%(pyver)s'), ('SciPy-Stack', '2020', '-Python-%(pyver)s'),
] ]
osdependencies = [('openssl')] osdependencies = [('openssl')]
skipsteps = ['configure'] skipsteps = ['configure']
buildopts = " USE_SYSTEM_GMP=1 USE_SYSTEM_CURL=1 USE_INTEL_MKL=1 " buildopts = " USE_SYSTEM_GMP=1 USE_INTEL_MKL=1 "
installopts = "prefix=%(installdir)s " installopts = "prefix=%(installdir)s "
local_julia_depot_path = "%(installdir)s/share/julia/site/" system_name = local_os.environ['SYSTEMNAME']
modextrapaths = { # arch_name = 'gpu'
'PATH': 'bin',
'JULIA_DEPOT_PATH': 'share/julia/site',
}
postinstallcmds = [
# prepare install env
(
'{ cat > %%(builddir)s/env.sh; } << \'EOF\'\n'
'#!/bin/bash\n'
'export JULIA_DEPOT_PATH=%s\n'
'EOF'
) % (local_julia_depot_path),
# installs extra packages exts_defaultclass = 'JuliaPackage'
( exts_list = [
'{ cat > %(builddir)s/pkg_add.jl; } << \'EOF\'\n'
'using Pkg; \n'
# General Purpose # General Purpose
'Pkg.add(name="PackageCompiler", version="1.2.5"); \n' ('PackageCompiler.jl', '1.2.5', {
'Pkg.add(name="HTTP", version="0.9.5"); \n' 'source_tmpl': 'v1.2.5.tar.gz',
'Pkg.add(name="Parsers", version="1.1.0"); \n' 'source_urls': ['https://github.com/JuliaLang/PackageCompiler.jl/archive/'],
'Pkg.add(name="VersionParsing", version="1.2.0"); \n' # 'packagespec': 'name="PackageCompiler", version="1.2.5"',
'Pkg.add(name="JSON", version="0.21.1"); \n' }),
'Pkg.add(name="WebIO", version="0.8.15"); \n' ('HTTP.jl', '0.9.5', {
'Pkg.add(name="ProgressMeter", version="1.5.0"); \n' 'source_tmpl': 'v0.9.5.tar.gz',
'Pkg.add(name="Conda", version="1.5.2"); \n' 'source_urls': ['https://github.com/JuliaWeb/HTTP.jl/archive/'],
'Pkg.add(name="PyCall", version="1.92.3"); \n' }),
'Pkg.add(name="LaTeXStrings", version="1.2.1"); \n' ('Parsers.jl', '1.1.0', {
'Pkg.add(name="DocumentFormat", version="3.2.0"); \n' 'source_tmpl': 'v1.1.0.tar.gz',
'source_urls': ['https://github.com/JuliaData/Parsers.jl/archive/'],
}),
('VersionParsing.jl', '1.2.0', {
'source_tmpl': 'v1.2.0.tar.gz',
'source_urls': ['https://github.com/JuliaInterop/VersionParsing.jl/archive/'],
}),
('JSON.jl', '0.21.1', {
'source_tmpl': 'v0.21.1.tar.gz',
'source_urls': ['https://github.com/JuliaIO/JSON.jl/archive/'],
}),
('WebIO.jl', '0.8.15', {
'source_tmpl': 'v0.8.15.tar.gz',
'source_urls': ['https://github.com/JuliaGizmos/WebIO.jl/archive/'],
}),
('ProgressMeter.jl', '1.5.0', {
'source_tmpl': 'v1.5.0.tar.gz',
'source_urls': ['https://github.com/timholy/ProgressMeter.jl/archive/'],
}),
('Conda.jl', '1.5.2', {
'source_tmpl': 'v1.5.2.tar.gz',
'source_urls': ['https://github.com/JuliaPy/Conda.jl/archive/'],
}),
('PyCall.jl', '1.92.3', {
'source_tmpl': 'v1.92.3.tar.gz',
'source_urls': ['https://github.com/JuliaPy/PyCall.jl/archive/'],
}),
('LaTeXStrings.jl', '1.2.1', {
'source_tmpl': 'v1.2.1.tar.gz',
'source_urls': ['https://github.com/stevengj/LaTeXStrings.jl/archive/'],
}),
('DocumentFormat.jl', '3.2.0', {
'source_tmpl': 'v3.2.0.tar.gz',
'source_urls': ['https://github.com/julia-vscode/DocumentFormat.jl/archive/'],
}),
# Data Science # Data Science
'Pkg.add(name="CSV", version="0.8.4"); \n' ('CSV.jl', '0.8.4', {
'Pkg.add(name="DataFrames", version="0.21.8"); \n' 'source_tmpl': 'v0.8.4.tar.gz',
'Pkg.add(name="Arrow", version="1.4.1"); \n' 'source_urls': ['https://github.com/JuliaData/CSV.jl/archive/'],
'Pkg.add(name="OnlineStats", version="1.5.8"); \n' }),
'Pkg.add(name="Query", version="1.0.0"); \n' ('DataFrames.jl', '0.21.8', {
'source_tmpl': 'v0.21.8.tar.gz',
'source_urls': ['https://github.com/JuliaData/DataFrames.jl/archive/'],
}),
('Arrow.jl', '1.4.1', {
'source_tmpl': 'v1.4.1.tar.gz',
'source_urls': ['https://github.com/JuliaData/Arrow.jl/archive/'],
}),
('OnlineStats.jl', '1.5.8', {
'source_tmpl': 'v1.5.8.tar.gz',
'source_urls': ['https://github.com/joshday/OnlineStats.jl/archive/'],
}),
('Query.jl', '1.0.0', {
'source_tmpl': 'v1.0.0.tar.gz',
'source_urls': ['https://github.com/queryverse/Query.jl/archive/'],
}),
# Scientific Domains # Scientific Domains
'Pkg.add(name="GSL", version="1.0.1"); \n' ('GSL.jl', '1.0.1', {
'Pkg.add(name="DifferentialEquations", version="6.16.0"); \n' 'source_tmpl': 'v1.0.1.tar.gz',
'Pkg.add(name="Distributions", version="0.24.18"); \n' 'source_urls': ['https://github.com/JuliaMath/GSL.jl/archive/refs/tags/'],
'Pkg.add(name="Optim", version="1.3.0"); \n' }),
'Pkg.add(name="IterativeSolvers", version="0.9.0"); \n' ('DifferentialEquations.jl', '6.16.0', {
'Pkg.add(name="AbstractFFTs", version="1.0.1"); \n' 'source_tmpl': 'v6.16.0.tar.gz',
'Pkg.add(name="ODE", version="2.13.0"); \n' 'source_urls': ['https://github.com/SciML/DifferentialEquations.jl/archive/'],
'Pkg.add(name="SpecialFunctions", version="1.3.0"); \n' }),
'Pkg.add(name="JuMP", version="0.21.7"); \n' ('Distributions.jl', '0.24.18', {
'source_tmpl': 'v0.24.18.tar.gz',
'source_urls': ['https://github.com/JuliaStats/Distributions.jl/archive/'],
}),
('Optim.jl', '1.3.0', {
'source_tmpl': 'v1.3.0.tar.gz',
'source_urls': ['https://github.com/JuliaNLSolvers/Optim.jl/archive/'],
}),
('IterativeSolvers.jl', '0.9.0', {
'source_tmpl': 'v0.9.0.tar.gz',
'source_urls': ['https://github.com/JuliaLinearAlgebra/IterativeSolvers.jl/archive/'],
}),
('AbstractFFTs.jl', '1.0.1', {
'source_tmpl': 'v1.0.1.tar.gz',
'source_urls': ['https://github.com/JuliaMath/AbstractFFTs.jl/archive/'],
}),
('OrdinaryDiffEq.jl', '5.52.7', {
'source_tmpl': 'v5.52.7.tar.gz',
'source_urls': ['https://github.com/SciML/OrdinaryDiffEq.jl/archive/'],
}),
('SpecialFunctions.jl', '1.3.0', {
'source_tmpl': 'v1.3.0.tar.gz',
'source_urls': ['https://github.com/JuliaMath/SpecialFunctions.jl/archive/'],
}),
('JuMP.jl', '0.21.7', {
'source_tmpl': 'v0.21.7.tar.gz',
'source_urls': ['https://github.com/jump-dev/JuMP.jl/archive/'],
}),
# Visualization # Visualization
'Pkg.add(name="GR", version="0.57.4"); \n' ('GR.jl', '0.57.4', {
'Pkg.add(name="PlotlyJS", version="0.14.1"); \n' 'source_tmpl': 'v0.57.4.tar.gz',
'Pkg.add(name="PyPlot", version="2.9.0"); \n' 'source_urls': ['https://github.com/jheinen/GR.jl/archive/'],
'Pkg.add(name="Plots", version="1.12.0"); \n' }),
'Pkg.add(name="UnicodePlots", version="1.3.0"); \n' ('PlotlyJS.jl', '0.14.1', {
'Pkg.add(name="StatsPlots", version="0.14.19"); \n' 'source_tmpl': 'v0.14.1.tar.gz',
# MachineLearning 'source_urls': ['https://github.com/JuliaPlots/PlotlyJS.jl/archive/'],
'Pkg.add(name="MLJ", version="0.16.2"); \n' }),
'Pkg.add(name="Flux", version="0.12.2"); \n' ('PyPlot.jl', '2.9.0', {
'Pkg.add(name="Knet", version="1.4.6"); \n' 'source_tmpl': 'v2.9.0.tar.gz',
'Pkg.add(name="MLDatasets", version="0.5.6"); \n' 'source_urls': ['https://github.com/JuliaPy/PyPlot.jl/archive/'],
'EOF' }),
), ('Plots.jl', '1.12.0', {
'source %(builddir)s/env.sh && %(installdir)s/bin/julia %(builddir)s/pkg_add.jl', 'source_tmpl': 'v1.12.0.tar.gz',
'source_urls': ['https://github.com/JuliaPlots/Plots.jl/archive/'],
# trigger precompilation }),
( ('UnicodePlots.jl', '1.3.0', {
'{ cat > %(builddir)s/precomp.jl; } << \'EOF\'\n' 'source_tmpl': 'v1.3.0.tar.gz',
# General Purpose 'source_urls': ['https://github.com/Evizero/UnicodePlots.jl/archive/'],
'using PackageCompiler; \n' }),
'using HTTP; \n' ('StatsPlots.jl', '0.14.19', {
'using Parsers; \n' 'source_tmpl': 'v0.14.19.tar.gz',
'using VersionParsing; \n' 'source_urls': ['https://github.com/JuliaPlots/StatsPlots.jl/archive/'],
'using JSON; \n' }),
'using WebIO; \n'
'using ProgressMeter; \n'
'using Conda; \n'
'using PyCall; \n'
'using LaTeXStrings; \n'
'using DocumentFormat; \n'
# Data Science
'using CSV; \n'
'using DataFrames; \n'
'using Arrow; \n'
'using OnlineStats; \n'
'using Query; \n'
# Scientific Domains
'using GSL; \n'
'using DifferentialEquations; \n'
'using Distributions; \n'
'using Optim; \n'
'using IterativeSolvers; \n'
'using AbstractFFTs; \n'
'using ODE; \n'
'using SpecialFunctions; \n'
'using JuMP; \n'
# Visualization
'using GR; \n'
'using PlotlyJS; \n'
'using PyPlot; \n'
'using Plots; \n'
'using UnicodePlots; \n'
'using StatsPlots; \n'
# MachineLearning
'using MLJ; \n'
'using Flux; \n'
'using Knet; \n'
'using MLDatasets; \n'
'EOF'
),
'source %(builddir)s/env.sh && %(installdir)s/bin/julia %(builddir)s/precomp.jl',
# adjust permissions of precompiled files
'for i in $(find %s); do chmod +r $i; done' % local_julia_depot_path,
] ]
sanity_check_paths = { sanity_check_paths = {
......
# For using $SYSTEMNAME to determine compute capability. The local prefix is to appease the checker
import os as local_os
name = 'Julia'
version = '1.6.1'
homepage = 'https://julialang.org/'
description = """Julia was designed from the beginning for high performance.
Julia programs compile to efficient native code for multiple platforms via LLVM
"""
site_contacts = 'j.goebbert@fz-juelich.de'
toolchain = {'name': 'gomkl', 'version': '2020'}
toolchainopts = {'usempi': True, 'pic': True}
source_urls = ['https://github.com/JuliaLang/julia/releases/download/v%(version)s/']
sources = ['julia-%(version)s-full.tar.gz']
builddependencies = [
('binutils', '2.34'),
('git', '2.28.0'),
('CMake', '3.18.0'),
]
dependencies = [
('Python', '3.8.5'),
('GMP', '6.2.0'),
('SciPy-Stack', '2020', '-Python-%(pyver)s', ('gcccoremkl', '9.3.0-2020.2.254')),
]
osdependencies = [('openssl')]
skipsteps = ['configure']
buildopts = " USE_SYSTEM_GMP=1 USE_INTEL_MKL=1 "
installopts = "prefix=%(installdir)s "
system_name = local_os.environ['SYSTEMNAME']
# arch_name = 'gpu'
exts_defaultclass = 'JuliaPackage'
exts_list = [
# General Purpose
('PackageCompiler.jl', '1.2.5', {
'source_tmpl': 'v1.2.5.tar.gz',
'source_urls': ['https://github.com/JuliaLang/PackageCompiler.jl/archive/'],
# 'packagespec': 'name="PackageCompiler", version="1.2.5"',
}),
('HTTP.jl', '0.9.5', {
'source_tmpl': 'v0.9.5.tar.gz',
'source_urls': ['https://github.com/JuliaWeb/HTTP.jl/archive/'],
}),
('Parsers.jl', '1.1.0', {
'source_tmpl': 'v1.1.0.tar.gz',
'source_urls': ['https://github.com/JuliaData/Parsers.jl/archive/'],
}),
('VersionParsing.jl', '1.2.0', {
'source_tmpl': 'v1.2.0.tar.gz',
'source_urls': ['https://github.com/JuliaInterop/VersionParsing.jl/archive/'],
}),
('JSON.jl', '0.21.1', {
'source_tmpl': 'v0.21.1.tar.gz',
'source_urls': ['https://github.com/JuliaIO/JSON.jl/archive/'],
}),
('WebIO.jl', '0.8.15', {
'source_tmpl': 'v0.8.15.tar.gz',
'source_urls': ['https://github.com/JuliaGizmos/WebIO.jl/archive/'],
}),
('ProgressMeter.jl', '1.5.0', {
'source_tmpl': 'v1.5.0.tar.gz',
'source_urls': ['https://github.com/timholy/ProgressMeter.jl/archive/'],
}),
('Conda.jl', '1.5.2', {
'source_tmpl': 'v1.5.2.tar.gz',
'source_urls': ['https://github.com/JuliaPy/Conda.jl/archive/'],
}),
('PyCall.jl', '1.92.3', {
'source_tmpl': 'v1.92.3.tar.gz',
'source_urls': ['https://github.com/JuliaPy/PyCall.jl/archive/'],
}),
('LaTeXStrings.jl', '1.2.1', {
'source_tmpl': 'v1.2.1.tar.gz',
'source_urls': ['https://github.com/stevengj/LaTeXStrings.jl/archive/'],
}),
('DocumentFormat.jl', '3.2.0', {
'source_tmpl': 'v3.2.0.tar.gz',
'source_urls': ['https://github.com/julia-vscode/DocumentFormat.jl/archive/'],
}),
# Data Science
('CSV.jl', '0.8.4', {
'source_tmpl': 'v0.8.4.tar.gz',
'source_urls': ['https://github.com/JuliaData/CSV.jl/archive/'],
}),
('DataFrames.jl', '0.21.8', {
'source_tmpl': 'v0.21.8.tar.gz',
'source_urls': ['https://github.com/JuliaData/DataFrames.jl/archive/'],
}),
('Arrow.jl', '1.4.1', {
'source_tmpl': 'v1.4.1.tar.gz',
'source_urls': ['https://github.com/JuliaData/Arrow.jl/archive/'],
}),
('OnlineStats.jl', '1.5.8', {
'source_tmpl': 'v1.5.8.tar.gz',
'source_urls': ['https://github.com/joshday/OnlineStats.jl/archive/'],
}),
('Query.jl', '1.0.0', {
'source_tmpl': 'v1.0.0.tar.gz',
'source_urls': ['https://github.com/queryverse/Query.jl/archive/'],
}),
# Scientific Domains
('GSL.jl', '1.0.1', {
'source_tmpl': 'v1.0.1.tar.gz',
'source_urls': ['https://github.com/JuliaMath/GSL.jl/archive/refs/tags/'],
}),
('DifferentialEquations.jl', '6.16.0', {
'source_tmpl': 'v6.16.0.tar.gz',
'source_urls': ['https://github.com/SciML/DifferentialEquations.jl/archive/'],
}),
('Distributions.jl', '0.24.18', {
'source_tmpl': 'v0.24.18.tar.gz',
'source_urls': ['https://github.com/JuliaStats/Distributions.jl/archive/'],
}),
('Optim.jl', '1.3.0', {
'source_tmpl': 'v1.3.0.tar.gz',
'source_urls': ['https://github.com/JuliaNLSolvers/Optim.jl/archive/'],
}),
('IterativeSolvers.jl', '0.9.0', {
'source_tmpl': 'v0.9.0.tar.gz',
'source_urls': ['https://github.com/JuliaLinearAlgebra/IterativeSolvers.jl/archive/'],
}),
('AbstractFFTs.jl', '1.0.1', {
'source_tmpl': 'v1.0.1.tar.gz',
'source_urls': ['https://github.com/JuliaMath/AbstractFFTs.jl/archive/'],
}),
('OrdinaryDiffEq.jl', '5.52.7', {
'source_tmpl': 'v5.52.7.tar.gz',
'source_urls': ['https://github.com/SciML/OrdinaryDiffEq.jl/archive/'],
}),
('SpecialFunctions.jl', '1.3.0', {
'source_tmpl': 'v1.3.0.tar.gz',
'source_urls': ['https://github.com/JuliaMath/SpecialFunctions.jl/archive/'],
}),
('JuMP.jl', '0.21.7', {
'source_tmpl': 'v0.21.7.tar.gz',
'source_urls': ['https://github.com/jump-dev/JuMP.jl/archive/'],
}),
# Visualization
('GR.jl', '0.57.4', {
'source_tmpl': 'v0.57.4.tar.gz',
'source_urls': ['https://github.com/jheinen/GR.jl/archive/'],
}),
('PlotlyJS.jl', '0.14.1', {
'source_tmpl': 'v0.14.1.tar.gz',
'source_urls': ['https://github.com/JuliaPlots/PlotlyJS.jl/archive/'],
}),
('PyPlot.jl', '2.9.0', {
'source_tmpl': 'v2.9.0.tar.gz',
'source_urls': ['https://github.com/JuliaPy/PyPlot.jl/archive/'],
}),
('Plots.jl', '1.12.0', {
'source_tmpl': 'v1.12.0.tar.gz',
'source_urls': ['https://github.com/JuliaPlots/Plots.jl/archive/'],
}),
('UnicodePlots.jl', '1.3.0', {
'source_tmpl': 'v1.3.0.tar.gz',
'source_urls': ['https://github.com/Evizero/UnicodePlots.jl/archive/'],
}),
('StatsPlots.jl', '0.14.19', {
'source_tmpl': 'v0.14.19.tar.gz',
'source_urls': ['https://github.com/JuliaPlots/StatsPlots.jl/archive/'],
}),
# MPI
('MPI.jl', '0.17.2', {
'mpiexec': 'srun',
'mpi_path': '$EBROOTOPENMPI',
'source_tmpl': 'v0.17.2.tar.gz',
'source_urls': ['https://github.com/JuliaParallel/MPI.jl/archive/'],
}),
]
modextravars = {
'JULIA_MPICC': 'mpicc',
'JULIA_MPIEXEC': 'srun',
# 'JULIA_MPIEXEC_ARGS': '',
'JULIA_MPI_ABI': 'OpenMPI',
'JULIA_MPI_BINARY': 'system',
'JULIA_MPI_PATH': '$::env(EBROOTOPENMPI)'
}
sanity_check_paths = {
'files': ['bin/julia', 'include/julia/julia.h', 'lib/libjulia.so'],
'dirs': ['bin', 'etc', 'include', 'lib', 'share']
}
moduleclass = 'lang'
easyblock = 'Bundle' easyblock = 'JuliaBundle'
name = 'JupyterKernel-Julia' name = 'JupyterKernel-Julia'
version = '1.6.1' version = '1.6.1'
...@@ -28,33 +28,47 @@ dependencies = [ ...@@ -28,33 +28,47 @@ dependencies = [
('Julia', version), ('Julia', version),
] ]
local_common_opts = { components = [
('julia', '0.5.6', {
'easyblock': 'PythonPackage',
'req_py_majver': '3', 'req_py_majver': '3',
'req_py_minver': '0' 'req_py_minver': '0',
}
exts_defaultclass = 'PythonPackage'
exts_filter = ('python -c "import %(ext_name)s"', '')
exts_default_options = {
'download_dep_fail': True,
'source_urls': [PYPI_SOURCE],
'use_pip': True, 'use_pip': True,
} 'sources': ['v%(version)s.tar.gz'],
'source_urls': ['https://github.com/JuliaPy/pyjulia/archive/'],
exts_list = [ 'checksums': [('sha256', 'ca4a1dc3df9b770dacbbecab5495cae817a5dde0ac2d3ff1db1f8e447f0e48b7')],
('julia', '0.5.6', dict(list(local_common_opts.items()) + [ 'download_dep_fail': True,
('checksums', [('sha256', '378d0377f75bb0e3bfc4cce19a56d3bf5a9a7be38e370e3a7cf3359bf4cd0378')]), 'start_dir': 'pyjulia-%(version)s',
('use_pip', True), }),
])),
] ]
local_jupyter_path = 'share/jupyter' local_jupyter_path = 'share/jupyter'
local_julia_depot_path = "%(installdir)s/share/julia/site/" # for Julia packages needed for Jupyter
exts_defaultclass = 'JuliaPackage'
exts_list = [
('ZMQ', '1.2.1', {
'source_tmpl': 'v1.2.1.tar.gz',
'source_urls': ['https://github.com/JuliaInterop/ZMQ.jl/archive/'],
}),
('IJulia', '1.23.2', {
# installs ijulia in JULIA_DEPOT_PATH and kernel in $JUPYTER_DATA_DIR/kernels
'source_tmpl': 'v1.23.2.tar.gz',
'source_urls': ['https://github.com/JuliaLang/IJulia.jl/archive/'],
'preinstallopts': 'export JUPYTER_DATA_DIR=%%(installdir)s/%s' % local_jupyter_path
}),
('Interact', '0.10.3', {
'source_tmpl': 'v0.10.3.tar.gz',
'source_urls': ['https://github.com/JuliaGizmos/Interact.jl/archive/'],
}),
('LanguageServer', '3.2.0', {
'source_tmpl': 'v3.2.0.tar.gz',
'source_urls': ['https://github.com/julia-vscode/LanguageServer.jl/archive/'],
}),
]
modextrapaths = { modextrapaths = {
'PYTHONPATH': ['lib/python%(pyshortver)s/site-packages'], 'PYTHONPATH': ['lib/python%(pyshortver)s/site-packages'],
'JUPYTER_PATH': ['share/jupyter'], # add search path for kernelspecs 'JUPYTER_PATH': [local_jupyter_path], # add search path for kernelspecs
'JULIA_DEPOT_PATH': ['share/julia/site'],
} }
# Ensure that the user-specific $HOME/.local/share/jupyter is always first entry in JUPYTHER_PATH # Ensure that the user-specific $HOME/.local/share/jupyter is always first entry in JUPYTHER_PATH
...@@ -65,43 +79,17 @@ prepend_path("JUPYTER_PATH", pathJoin(os.getenv("HOME"), ".local/share/jupyter") ...@@ -65,43 +79,17 @@ prepend_path("JUPYTER_PATH", pathJoin(os.getenv("HOME"), ".local/share/jupyter")
postinstallcmds = [ postinstallcmds = [
# Create virtual environment to ensure we install in the correct directory !!! # Create virtual environment to ensure we install in the correct directory !!!
'python3 -m venv %(installdir)s --system-site-packages', 'python3 -m venv %(installdir)s --system-site-packages',
'mkdir -p %%(installdir)s/%s' % local_jupyter_path,
( (
'{ cat > %%(builddir)s/env.sh; } << \'EOF\'\n' '{ cat > %%(builddir)s/env.sh; } << \'EOF\'\n'
'#!/bin/bash\n' '#!/bin/bash\n'
'source %%(installdir)s/bin/activate\n' 'source %%(installdir)s/bin/activate\n'
'export PYTHONPATH=%%(installdir)s/lib/python%%(pyshortver)s/site-packages:${PYTHONPATH}\n' 'export PYTHONPATH=%%(installdir)s/lib/python%%(pyshortver)s/site-packages:${PYTHONPATH}\n'
'' ''
'export JULIA_DEPOT_PATH=%s:${JULIA_DEPOT_PATH}\n' 'export JULIA_DEPOT_PATH=${EBJULIA_STD_DEPOT_PATH}:${EBJULIA_ADMIN_DEPOT_PATH}\n'
'export JUPYTER_DATA_DIR=%%(installdir)s/%s\n' 'export JUPYTER_DATA_DIR=%%(installdir)s/%s\n'
'EOF' 'EOF'
) % (local_julia_depot_path, local_jupyter_path), ) % (local_jupyter_path),
# installs ijulia in JULIA_DEPOT_PATH and kernel in $JUPYTER_DATA_DIR/kernels
(
'{ cat > %(builddir)s/pkg_add.jl; } << \'EOF\'\n'
'using Pkg; \n'
'Pkg.add(name="ZMQ", version="1.2.1"); \n'
'Pkg.add(name="IJulia", version="1.23.2"); \n'
'Pkg.add(name="Interact", version="0.10.3"); \n'
'Pkg.add(name="LanguageServer", version="3.2.0"); \n'
'Pkg.build("IJulia"); \n'
'EOF'
),
'source %(builddir)s/env.sh && julia %(builddir)s/pkg_add.jl',
# to trigger the precompilation
(
'{ cat > %(builddir)s/precomp.jl; } << \'EOF\'\n'
'using ZMQ; \n'
'using IJulia; \n'
'using Interact; \n'
'using LanguageServer; \n'
'EOF'
),
'source %(builddir)s/env.sh && julia %(builddir)s/precomp.jl',
# adjust permissions of precompiled files
'for i in $(find %s); do chmod +r $i; done' % local_julia_depot_path,
# configure Python<->Julia bridge (of python package julia) # configure Python<->Julia bridge (of python package julia)
'source %(builddir)s/env.sh && python -c "import julia; julia.install()"', 'source %(builddir)s/env.sh && python -c "import julia; julia.install()"',
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment