diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3e08d14d25f20d589e6f260f1d53efffc64a2ad3..336030fbd83ae137bf14491c8a11ed2661db0e1a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,4 +3,4 @@ repos:
rev: '' # Use the sha / tag you want to point at
hooks:
- id: autopep8
- args: [-v]
\ No newline at end of file
+ args: [-v,--in-place]
\ No newline at end of file
diff --git a/Custom_EasyBlocks/bazel.py b/Custom_EasyBlocks/bazel.py
index b0ad778f476973b0d1ca96631142bca216379b0d..6e21bd1c0c0c93d9ae428e2eefb0420a1dd2f9a4 100644
--- a/Custom_EasyBlocks/bazel.py
+++ b/Custom_EasyBlocks/bazel.py
@@ -35,6 +35,157 @@ from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which
from easybuild.tools.modules import get_software_root, get_software_version
from easybuild.tools.run import run_cmd
+from easybuild.framework.easyconfig import CUSTOM
+
+
+class EB_Bazel(EasyBlock):
+ """Support for building/installing Bazel."""
+
+ @staticmethod
+ def extra_options():
+ """Extra easyconfig parameters specific to EB_Bazel."""
+ extra_vars = {
+ 'static': [False, "Build statically linked executables", CUSTOM],
+ }
+ return EasyBlock.extra_options(extra_vars)
+
+ def fixup_hardcoded_paths(self):
+ """Patch out hard coded paths to compiler and binutils tools"""
+ binutils_root = get_software_root('binutils')
+ gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
+ gcc_ver = get_software_version(
+ 'GCCcore') or get_software_version('GCC')
+
+ # only patch Bazel scripts if binutils & GCC installation prefix could be determined
+ if not binutils_root or not gcc_root:
+ self.log.info(
+ "Not patching Bazel build scripts, installation prefix for binutils/GCC not found")
+ return
+
+ # replace hardcoded paths in (unix_)cc_configure.bzl
+ # hard-coded paths in (unix_)cc_configure.bzl were removed in 0.19.0
+ if LooseVersion(self.version) < LooseVersion('0.19.0'):
+ regex_subs = [
+ (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
+ (r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
+ ]
+ for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
+ filepath = os.path.join('tools', 'cpp', conf_bzl)
+ if os.path.exists(filepath):
+ apply_regex_substitutions(filepath, regex_subs)
+
+ # replace hardcoded paths in CROSSTOOL
+ # CROSSTOOL script is no longer there in Bazel 0.24.0
+ if LooseVersion(self.version) < LooseVersion('0.24.0'):
+ res = glob.glob(os.path.join(gcc_root, 'lib',
+ 'gcc', '*', gcc_ver, 'include'))
+ if res and len(res) == 1:
+ gcc_lib_inc = res[0]
+ else:
+ raise EasyBuildError(
+ "Failed to pinpoint location of GCC include files: %s", res)
+
+ gcc_lib_inc_bis = os.path.join(
+ os.path.dirname(gcc_lib_inc), 'include-fixed')
+ if not os.path.exists(gcc_lib_inc_bis):
+ self.log.info(
+ "Derived directory %s does not exist, falling back to %s", gcc_lib_inc_bis, gcc_lib_inc)
+ gcc_lib_inc_bis = gcc_lib_inc
+
+ gcc_cplusplus_inc = os.path.join(
+ gcc_root, 'include', 'c++', gcc_ver)
+ if not os.path.exists(gcc_cplusplus_inc):
+ raise EasyBuildError(
+ "Derived directory %s does not exist", gcc_cplusplus_inc)
+
+ regex_subs = [
+ (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
+ (r'(cxx_builtin_include_directory:.*)/usr/lib/gcc',
+ r'\1%s' % gcc_lib_inc),
+ (r'(cxx_builtin_include_directory:.*)/usr/local/include',
+ r'\1%s' % gcc_lib_inc_bis),
+ (r'(cxx_builtin_include_directory:.*)/usr/include',
+ r'\1%s' % gcc_cplusplus_inc),
+ ]
+ for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
+ path = which(tool)
+ if path:
+ regex_subs.append(
+ (os.path.join('/usr', 'bin', tool), path))
+ else:
+ raise EasyBuildError(
+ "Failed to determine path to '%s'", tool)
+
+ apply_regex_substitutions(os.path.join(
+ 'tools', 'cpp', 'CROSSTOOL'), regex_subs)
+
+ def configure_step(self):
+ """Custom configuration procedure for Bazel."""
+
+ # Last instance of hardcoded paths was removed in 0.24.0
+ if LooseVersion(self.version) < LooseVersion('0.24.0'):
+ self.fixup_hardcoded_paths()
+
+ # enable building in parallel
+ bazel_args = '--jobs=%d' % self.cfg['parallel']
+
+ # Bazel provides a JDK by itself for some architectures
+ # We want to enforce it using the JDK we provided via modules
+ # This is required for Power where Bazel does not have a JDK, but requires it for building itself
+ # See https://github.com/bazelbuild/bazel/issues/10377
+ bazel_args += ' --host_javabase=@local_jdk//:jdk'
+
+ env.setvar('EXTRA_BAZEL_ARGS', bazel_args)
+
+ def build_step(self):
+ """Custom build procedure for Bazel."""
+ static = ''
+ if self.cfg['static'] is True:
+ static = 'export BAZEL_LINKOPTS=-static-libstdc++:-static-libgcc BAZEL_LINKLIBS=-l%:libstdc++.a:-lm && '
+
+ cmd = '%s %s ./compile.sh' % (static, self.cfg['prebuildopts'])
+ run_cmd(cmd, log_all=True, simple=True, log_ok=True)
+
+ def install_step(self):
+ """Custom install procedure for Bazel."""
+ copy_file(os.path.join('output', 'bazel'),
+ os.path.join(self.installdir, 'bin', 'bazel'))
+
+ def sanity_check_step(self):
+ """Custom sanity check for Bazel."""
+ custom_paths = {
+ 'files': ['bin/bazel'],
+ 'dirs': [],
+ }
+ super(EB_Bazel, self).sanity_check_step(custom_paths=custom_paths)
+
+
+# Copyright 2009-2020 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for building and installing Bazel, implemented as an easyblock
+"""
class EB_Bazel(EasyBlock):
@@ -45,25 +196,32 @@ class EB_Bazel(EasyBlock):
binutils_root = get_software_root('binutils')
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
- gcc_ver = get_software_version('GCCcore') or get_software_version('GCC')
+ gcc_ver = get_software_version(
+ 'GCCcore') or get_software_version('GCC')
# only patch Bazel scripts if binutils & GCC installation prefix could be determined
if binutils_root and gcc_root:
- res = glob.glob(os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
+ res = glob.glob(os.path.join(gcc_root, 'lib',
+ 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
- raise EasyBuildError("Failed to pinpoint location of GCC include files: %s", res)
+ raise EasyBuildError(
+ "Failed to pinpoint location of GCC include files: %s", res)
- gcc_lib_inc_bis = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed')
+ gcc_lib_inc_bis = os.path.join(
+ os.path.dirname(gcc_lib_inc), 'include-fixed')
if not os.path.exists(gcc_lib_inc_bis):
- self.log.info("Derived directory %s does not exist, falling back to %s", gcc_lib_inc_bis, gcc_lib_inc)
+ self.log.info(
+ "Derived directory %s does not exist, falling back to %s", gcc_lib_inc_bis, gcc_lib_inc)
gcc_lib_inc_bis = gcc_lib_inc
- gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver)
+ gcc_cplusplus_inc = os.path.join(
+ gcc_root, 'include', 'c++', gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
- raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc)
+ raise EasyBuildError(
+ "Derived directory %s does not exist", gcc_cplusplus_inc)
# replace hardcoded paths in CROSSTOOL
@@ -71,18 +229,24 @@ class EB_Bazel(EasyBlock):
if LooseVersion(self.version) < LooseVersion('0.24.0'):
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
- (r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc),
- (r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_bis),
- (r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc),
+ (r'(cxx_builtin_include_directory:.*)/usr/lib/gcc',
+ r'\1%s' % gcc_lib_inc),
+ (r'(cxx_builtin_include_directory:.*)/usr/local/include',
+ r'\1%s' % gcc_lib_inc_bis),
+ (r'(cxx_builtin_include_directory:.*)/usr/include',
+ r'\1%s' % gcc_cplusplus_inc),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
path = which(tool)
if path:
- regex_subs.append((os.path.join('/usr', 'bin', tool), path))
+ regex_subs.append(
+ (os.path.join('/usr', 'bin', tool), path))
else:
- raise EasyBuildError("Failed to determine path to '%s'", tool)
+ raise EasyBuildError(
+ "Failed to determine path to '%s'", tool)
- apply_regex_substitutions(os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
+ apply_regex_substitutions(os.path.join(
+ 'tools', 'cpp', 'CROSSTOOL'), regex_subs)
# replace hardcoded paths in (unix_)cc_configure.bzl
regex_subs = [
@@ -94,7 +258,8 @@ class EB_Bazel(EasyBlock):
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
else:
- self.log.info("Not patching Bazel build scripts, installation prefix for binutils/GCC not found")
+ self.log.info(
+ "Not patching Bazel build scripts, installation prefix for binutils/GCC not found")
# enable building in parallel
bazel_args = '--jobs=%d' % self.cfg['parallel']
@@ -114,7 +279,8 @@ class EB_Bazel(EasyBlock):
def install_step(self):
"""Custom install procedure for Bazel."""
- copy_file(os.path.join('output', 'bazel'), os.path.join(self.installdir, 'bin', 'bazel'))
+ copy_file(os.path.join('output', 'bazel'),
+ os.path.join(self.installdir, 'bin', 'bazel'))
def sanity_check_step(self):
"""Custom sanity check for Bazel."""
diff --git a/Custom_EasyBlocks/jax.py b/Custom_EasyBlocks/jax.py
index d4d9febd920033297a8202eb00f42073aacf4fd0..768dc527db871b12fab57e17b79e5ff55aae97ae 100644
--- a/Custom_EasyBlocks/jax.py
+++ b/Custom_EasyBlocks/jax.py
@@ -43,6 +43,7 @@ class EB_JAX(EasyBlock):
bazel)
# Tell Bazel to pass PYTHONPATH through to what it's building, so it can find scipy etc.
self.cfg['buildopts'] += '--bazel_options=--action_env=PYTHONPATH '
+ self.cfg['buildopts'] += '--bazel_options=--action_env=PATH '
self.cfg['buildopts'] += '--cuda_compute_capabilities "7.0,7.5,8.0" '
self.cfg['buildopts'] += '--noenable_mkl_dnn '
if get_cpu_architecture() == POWER:
diff --git a/Golden_Repo/b/Bazel/Bazel-3.6.0-GCCcore-9.3.0.eb b/Golden_Repo/b/Bazel/Bazel-3.6.0-GCCcore-9.3.0.eb
index 3bcaab02204341421c1cefaf3da1985e5d3a1a61..aa34022f27838d2ffe9574578030c5186bf70465 100644
--- a/Golden_Repo/b/Bazel/Bazel-3.6.0-GCCcore-9.3.0.eb
+++ b/Golden_Repo/b/Bazel/Bazel-3.6.0-GCCcore-9.3.0.eb
@@ -13,6 +13,12 @@ source_urls = ['https://github.com/bazelbuild/bazel/releases/download/%(version)
sources = ['%(namelower)s-%(version)s-dist.zip']
patches = ['%(name)s-3.4.1-fix-grpc-protoc.patch']
+# Bazel doesn't pass LD_LIBRARY_PATH etc. through, even to its own "process-wrapper".
+# So we'll statically link bazel, to avoid errors like:
+# "process-wrapper: /lib64/libstdc++.so.6: version `GLIBCXX_3.4.21' not found"
+# See https://github.com/bazelbuild/bazel/issues/4137
+static = True
+
builddependencies = [
('binutils', '2.34'),
('Python', '3.8.5'),