Commit 2ca10719 authored by Stepan Nassyr's avatar Stepan Nassyr
Browse files

ray builds now (with a hack)

parent e52b9778
easyblock = 'CMakeNinja'
name = 'Arrow'
version = '7.0.0'
versionsuffix = '-Python-%(pyver)s'
homepage = 'https://arrow.apache.org'
description = """Apache Arrow (incl. PyArrow Python bindings), a cross-language development platform
for in-memory data."""
toolchain = {'name': 'goolf', 'version': '2021a.11'}
source_urls = ['https://archive.apache.org/dist/%(namelower)s/%(namelower)s-%(version)s']
sources = ['apache-arrow-%(version)s.tar.gz']
checksums = ['e8f49b149a15ecef4e40fcfab1b87c113c6b1ee186005c169e5cdf95d31a99de']
builddependencies = [
('CMake', '3.20.0'),
('Ninja', '1.10.2'),
('Autotools', '20210330'),
('flex', '2.6.4'),
('Bison', '3.7.6'),
('pkg-config', '0.29.2'),
]
# Arrow strongly prefers included jemalloc, so not including it as a dependency
dependencies = [
('Python', '3.9.4'),
('SciPy-Stack', '2021a', versionsuffix),
('Boost', '1.75.0', versionsuffix),
]
start_dir = 'cpp'
# see https://arrow.apache.org/docs/developers/python.html
configopts = "-DARROW_PYTHON=on -DARROW_PARQUET=ON -DARROW_WITH_SNAPPY=ON "
configopts += "-DCMAKE_INSTALL_LIBDIR=lib -DPython3_ROOT_DIR=$EBROOTPYTHON"
# also install Python bindings
local_install_pyarrow_cmds = "export PKG_CONFIG_PATH=%(installdir)s/lib/pkgconfig:$PKG_CONFIG_PATH && "
local_install_pyarrow_cmds += "export PYTHONPATH=%(installdir)s/lib/python%(pyshortver)s/site-packages:$PYTHONPATH && "
local_install_pyarrow_cmds += "cd %(builddir)s/*arrow-%(version)s/python && export XDG_CACHE_HOME=$TMPDIR && "
local_install_pyarrow_cmds += "sed -i 's/numpy==[0-9.]*/numpy/g' pyproject.toml && "
local_install_pyarrow_cmds += "PYARROW_WITH_PARQUET=1 pip install --prefix %(installdir)s ."
postinstallcmds = [local_install_pyarrow_cmds]
modextrapaths = {'PYTHONPATH': 'lib/python%(pyshortver)s/site-packages'}
sanity_check_paths = {
'files': ['lib/libarrow.a', 'lib/libarrow.%s' % SHLIB_EXT,
'lib/libarrow_python.a', 'lib/libarrow_python.%s' % SHLIB_EXT],
'dirs': ['include/arrow', 'lib/cmake/arrow', 'lib/pkgconfig', 'lib/python%(pyshortver)s/site-packages'],
}
sanity_check_commands = [
"python -c 'import pyarrow'",
"python -c 'import pyarrow.parquet'",
]
moduleclass = 'data'
easyblock = 'ConfigureMake'
name = 'nodejs'
version = '16.15.0'
local_libversion = '93'
homepage = 'http://%(name)s.org'
description = """Node.js is a platform built on Chrome's JavaScript runtime
for easily building fast, scalable network applications. Node.js uses an
event-driven, non-blocking I/O model that makes it lightweight and efficient,
perfect for data-intensive real-time applications that run across distributed devices."""
toolchain = {'name': 'GCCcore', 'version': '11.1.0'}
source_urls = ['http://%(name)s.org/dist/v%(version)s/']
sources = ['node-v%(version)s.tar.gz']
checksums = ['38e041b7249afb331663613f860b84b51852efa02b4ebff852f9610bef680d7d']
builddependencies = [('binutils', '2.36.1')]
# Python is required (only) as build dependency
allow_system_deps = [('Python', SYS_PYTHON_VERSION)]
configopts = [
'--with-intl=none', # Fully disable ICU to avoid issues with the embedded icu-small library
'--shared --with-intl=none', # Build libnode.so in a second run
]
# Link libv8 libs to libnode
postinstallcmds = [
"cd %%(installdir)s/lib; ln -s libnode.%s.%s libnode.%s" % (SHLIB_EXT, local_libversion, SHLIB_EXT),
"cd %%(installdir)s/lib; ln -s libnode.%s.%s libv8.%s" % (SHLIB_EXT, local_libversion, SHLIB_EXT),
"cd %%(installdir)s/lib; ln -s libnode.%s.%s libv8_libbase.%s" % (SHLIB_EXT, local_libversion, SHLIB_EXT),
"cd %%(installdir)s/lib; ln -s libnode.%s.%s libv8_libplatform.%s" % (SHLIB_EXT, local_libversion, SHLIB_EXT),
]
sanity_check_paths = {
'files': ['bin/node', 'bin/npm', 'lib/libnode.%s.%s' % (SHLIB_EXT, local_libversion)],
'dirs': ['lib/node_modules', 'include/node']
}
modextrapaths = {'CPATH': 'include/node'}
moduleclass = 'lang'
# Author: Pavel Grochal (INUITS)
# License: GPLv2
easyblock = 'PythonBundle'
name = 'ray-deps'
version = '1.12.0'
versionsuffix = '-Python-%(pyver)s'
homepage = "https://docs.ray.io/en/latest/"
description = "Python packages needed by ray-project."
toolchain = {'name': 'GCCcore', 'version': '11.1.0'}
builddependencies = [
('CMake', '3.20.0'),
('binutils', '2.36.1'),
]
dependencies = [
('Python', '3.9.4'),
# ('SciPy-Stack', '2021a', versionsuffix),
('protobuf', '3.16.0'),
]
use_pip = True
sanity_pip_check = True
exts_list = [
('aiosignal', '1.2.0', {
'checksums': ['78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2'],
}),
('frozenlist', '1.3.0', {
'checksums': ['ce6f2ba0edb7b0c1d8976565298ad2deba6f8064d2bebb6ffce2ca896eb35b0b'],
}),
('grpcio', '1.43.0', {
'modulename' : 'grpc',
'checksums': ['735d9a437c262ab039d02defddcb9f8f545d7009ae61c0114e19dda3843febe5'],
}),
]
moduleclass = 'lib'
# Author: Pavel Grochal (INUITS)
# License: GPLv2
easyblock = 'PythonBundle'
# Author: 2022 Stepan Nassyr (Forschungszentrum Juelich)
#
# WARNING: This will fail to compile due to use of 'env -' by bazel and not propagating the
# environment to protoc calls The proper solution would be to patch Bazel or ray to
# propagate the environment, but I just did the following workaround:
#
# sudo mkdir /usr/libreplace
# sudo mkdir /usr/liborig
# (as root) cat "/usr/libreplace" >> /etc/ld.so.conf.d/raybuildlibs.conf
# module load GCC/<version>
# sudo cp $EBROOTGCC/lib64/libstdc++*so* /usr/libreplace/
# sudo cp <any other libraries it doesn't find> /usr/libreplace/
# sudo mv /usr/lib64/libstdc++*so* /usr/liborig
# sudo ldconfig
# <build ray>
# sudo mv /usr/liborig/libstdc++*so* /usr/lib64
# sudo rm /etc/ld.so.conf.d/raybuildlibs.conf
# sudo ldconfig
#
# This is obviously an ugly hack, but I can't invest more time into this right now
#
easyblock = 'PythonPackage'
name = 'ray-project'
version = '1.12.0'
versionsuffix = '-Python-%(pyver)s'
homepage = "https://docs.ray.io/en/latest/"
description = "Ray is a fast and simple framework for building and running distributed applications."
homepage = "https://www.ray.io/"
description = "An open source framework that provides a simple, universal API for building distributed applications. Ray is packaged with RLlib, a scalable reinforcement learning library, and Tune, a scalable hyperparameter tuning library."
toolchain = {'name': 'goolf', 'version': '2021a.11'}
toolchainopts = {'usempi': True}
source_urls = ['https://github.com/%(name)s/ray/archive/refs/tags/']
sources = ['ray-%(version)s.tar.gz']
checksums = ['f358a942c35890c8939706ef0362569bcd3aca225109b6875492c4832caf2fe7']
patches = ['ray-project-1.12.0-bazelrc-env.patch']
builddependencies = [
('Bazel', '5.1.1')
]
dependencies = [
('Python', '3.9.4'),
('Arrow', '7.0.0', versionsuffix),
('ray-deps', version, versionsuffix),
('SciPy-Stack', '2021a', versionsuffix),
('PyYAML', '5.4.1', versionsuffix),
('dm-tree', '0.1.7', versionsuffix),
('protobuf', '3.16.0'),
('gRPC', '1.37.1'),
('nodejs', '16.15.0'),
# ('gRPC', '1.37.1'),
('lz4', '1.9.3'),
]
use_pip = True
sanity_pip_check = True
options = {'modulename': 'ray'}
# Ray is run from wheel, because build doesn't pass LD_LIBRARY_PATH properly to Bazel/Protobuf
# https://github.com/ray-project/ray/issues/5274
exts_list = [
# ('hiredis', '2.0.0', {
# 'checksums': ['81d6d8e39695f2c37954d1011c0480ef7cf444d4e3ae24bc5e89ee5de360139a'],
# }),
# ('aioredis', '2.0.1', {
# 'checksums': ['eaa51aaf993f2d71f54b70527c440437ba65340588afeb786cd87c55c89cd98e'],
# }),
# ('blessings', '1.7', {
# 'checksums': ['98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d'],
# }),
# ('gpustat', '0.6.0', {
# 'checksums': ['f69135080b2668b662822633312c2180002c10111597af9631bb02e042755b6c'],
# }),
# ('prometheus-client', '0.9.0', {
# 'source_tmpl': 'prometheus_client-%(version)s.tar.gz',
# 'checksums': ['9da7b32f02439d8c04f7777021c304ed51d9ec180604700c1ba72a4d44dceb03'],
# }),
# ('cachetools', '4.1.1', {
# 'checksums': ['bbaa39c3dede00175df2dc2b03d0cf18dd2d32a7de7beb68072d13043c9edb20'],
# }),
# ('pyasn1-modules', '0.2.8', {
# 'checksums': ['905f84c712230b2c592c19470d3ca8d552de726050d1d1716282a1f6146be65e'],
# }),
# ('rsa', '4.6', {
# 'checksums': ['109ea5a66744dd859bf16fe904b8d8b627adafb9408753161e766a92e7d681fa'],
# }),
# ('google-auth', '1.23.0', {
# 'modulename': 'google.auth',
# 'checksums': ['5176db85f1e7e837a646cd9cede72c3c404ccf2e3373d9ee14b2db88febad440'],
# }),
# ('googleapis-common-protos', '1.52.0', {
# 'modulename': 'google.api',
# 'checksums': ['560716c807117394da12cecb0a54da5a451b5cf9866f1d37e9a5e2329a665351'],
# }),
# ('google-api-core', '1.19.1', {
# 'modulename': 'google.api_core',
# 'checksums': ['6ce7609320449ad8680312395b89396a7d6951032f1886bb1a16534e4ff3867d'],
# }),
# ('opencensus', '0.7.11', {
# 'installopts': " context/opencensus-context/",
# 'source_tmpl': 'v%(version)s.tar.gz',
# 'source_urls': ['https://github.com/census-instrumentation/opencensus-python/archive'],
# 'checksums': ['af3a4f7f8782ec38bda43aabb747d29714af4ac5d75219b9f5970ec6d17cef00'],
# }),
# ('lz4', '3.1.0', {
# 'checksums': ['debe75513db3eb9e5cdcd82a329ff38374b6316ab65b848b571e0404746c1e05'],
# }),
# ('async-timeout', '3.0.1', {
# 'checksums': ['0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f'],
# }),
# ('multidict', '5.0.2', {
# 'checksums': ['e5bf89fe57f702a046c7ec718fe330ed50efd4bcf74722940db2eb0919cddb1c'],
# }),
# ('typing-extensions', '3.7.4.3', {
# 'modulename': 'typing_extensions',
# 'source_tmpl': 'typing_extensions-%(version)s.tar.gz',
# 'checksums': ['99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c'],
# }),
# ('yarl', '1.6.3', {
# 'checksums': ['8a9066529240171b68893d60dca86a763eae2139dd42f42106b03cf4b426bf10'],
# }),
# ('colorful', '0.5.4', {
# 'checksums': ['86848ad4e2eda60cd2519d8698945d22f6f6551e23e95f3f14dfbb60997807ea'],
# }),
# ('colorama', '0.4.4', {
# 'checksums': ['5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b'],
# }),
# ('filelock', '3.0.12', {
# 'checksums': ['18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59'],
# }),
# ('google', '3.0.0', {
# 'checksums': ['143530122ee5130509ad5e989f0512f7cb218b2d4eddbafbad40fd10e8d8ccbe'],
# }),
# ('soupsieve', '2.0.1', {
# 'checksums': ['a59dc181727e95d25f781f0eb4fd1825ff45590ec8ff49eadfd7f1a537cc0232'],
# }),
# ('beautifulsoup4', '4.9.3', {
# 'modulename': 'bs4',
# 'checksums': ['84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25'],
# }),
# ('grpcio', '1.33.2', {
# 'modulename': 'grpc',
# 'checksums': ['21265511880056d19ce4f809ce3fbe2a3fa98ec1fc7167dbdf30a80d3276202e'],
# 'preinstallopts': "export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=%(parallel)s && ",
# }),
# ('jsonschema', '3.2.0', {
# 'checksums': ['c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a'],
# }),
# ('pyrsistent', '0.17.3', {
# 'checksums': ['2e636185d9eb976a18a8a8e96efce62f2905fea90041958d8cc2a189756ebf3e'],
# }),
# ('msgpack', '1.0.0', {
# 'checksums': ['9534d5cc480d4aff720233411a1f765be90885750b07df772380b34c10ecb5c0'],
# }),
# ('py-spy', '0.3.3', {
# 'modulename': False,
# 'source_tmpl': 'py_spy-%(version)s-py2.py3-none-manylinux1_x86_64.whl',
# 'checksums': ['e9d6946741c267fe82aef18d2fc1e095a90a83fb5f3d9fc89b0f20a39613a639'],
# }),
# ('redis', '3.4.1', {
# 'checksums': ['0dcfb335921b88a850d461dc255ff4708294943322bd55de6cfd68972490ca1f'],
# }),
('ray', version, {
'source_urls' : [GITHUB_SOURCE],
'checksums': ['f358a942c35890c8939706ef0362569bcd3aca225109b6875492c4832caf2fe7'],
'patches': ['ray-project-1.12.0-bazelrc-env.patch'],
'preinstallopts' : 'cd python && ',
'use_pip_editable' : True,
}),
]
preinstallopts = 'cd dashboard/client && '
preinstallopts += 'npm install && '
preinstallopts += 'npm run build && '
# Bazel picks up the headers with the patches but still insists on building it's own versions which then conflict
# with the modules. I don't know how to make Bazel use these libraries, so just unload them and let it build them
preinstallopts += 'module unload nodejs && '
preinstallopts += 'module unload protobuf && '
preinstallopts += 'module unload Boost && '
preinstallopts += 'cd ../../python && BAZEL_LIMIT_CPUS=$EASYBUILD_PARALLEL '
postinstallcmds = [
"""export eggdest=$(cat %(installdir)s/lib/python%(pyshortver)s/site-packages/ray.egg-link | head -n1);
cp -r $eggdest/ray* %(installdir)s/lib/python%(pyshortver)s/site-packages/;
rm %(installdir)s/lib/python%(pyshortver)s/site-packages/ray/{rllib,dashboard};
cp -r $eggdest/{../dashboard,../rllib} %(installdir)s/lib/python%(pyshortver)s/site-packages/ray/;
rm %(installdir)s/lib/python%(pyshortver)s/site-packages/ray.egg-link"""]
use_pip_editable = True
sanity_check_paths = {
'files': ['bin/ray'],
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment