Skip to content
Snippets Groups Projects
Commit 26bf2d1f authored by Damian Alvarez's avatar Damian Alvarez
Browse files

Merge branch 'bazel' into '2022'

Bazel 3.7.2 and gcc11 patches

See merge request hps-public/easybuild-repository!887
parents e1cee14e 3ea19063
Branches
No related tags found
No related merge requests found
Showing
with 455 additions and 89 deletions
From cd3c41eb5a29ca475b7bafc42aa71e94363d46df Mon Sep 17 00:00:00 2001
From: Alexander Grund <alexander.grund@tu-dresden.de>
Date: Tue, 28 Jul 2020 19:51:13 +0200
Subject: [PATCH] Fix environment for protobuf compilation in grpc
Add use_default_shell_env = True to protoc invocation for grpc to mirror
what the protobuf cc_proto_library & co are doing
Fixes a failure in invocing protoc when it is build in a non-default
environment (e.g. with a custom LD_LIBRARY_PATH)
Fixes #11852, fixes #11855
---
third_party/grpc/bazel/generate_cc.bzl | 1 +
1 file changed, 1 insertion(+)
diff --git a/third_party/grpc/bazel/generate_cc.bzl b/third_party/grpc/bazel/generate_cc.bzl
index 38a5b460f90..d5a4e27bc88 100644
--- a/third_party/grpc/bazel/generate_cc.bzl
+++ b/third_party/grpc/bazel/generate_cc.bzl
@@ -123,6 +123,7 @@ def generate_cc_impl(ctx):
outputs = out_files,
executable = ctx.executable.protoc,
arguments = arguments,
+ use_default_shell_env = True,
)
return struct(files = depset(out_files))
diff --git a/third_party/protobuf/3.13.0.patch b/third_party/protobuf/3.13.0.patch
index bde8684b82..3336ef4024 100644
--- a/third_party/protobuf/3.13.0.patch
+++ b/third_party/protobuf/3.13.0.patch
@@ -38,3 +38,15 @@ index cfdb28e2e..3705fdbe3 100644
+ "@io_bazel//third_party:gson",
],
)
+diff --git a/protobuf.bzl b/protobuf.bzl
+index 050eafc54..12d3edb94 100644
+--- a/protobuf.bzl
++++ b/protobuf.bzl
+@@ -352,6 +352,7 @@ def _internal_gen_well_known_protos_java_impl(ctx):
+ inputs = descriptors,
+ outputs = [srcjar],
+ arguments = [args],
++ use_default_shell_env = True,
+ )
+
+ return [
name = 'Bazel'
version = '3.7.2'
homepage = 'https://bazel.io/'
description = """Bazel is a build tool that builds code quickly and reliably.
It is used to build the majority of Google's software."""
toolchain = {'name': 'GCCcore', 'version': '11.2.0'}
source_urls = [
'https://github.com/bazelbuild/%(namelower)s/releases/download/%(version)s']
sources = ['%(namelower)s-%(version)s-dist.zip']
patches = [
'%(name)s-3.4.1-fix-grpc-protoc.patch',
'java15.patch',
'%(name)s-3.7.1_fix-protobuf-env.patch',
'gcc11-bazel-3.7.2.patch',
'gcc11-2-bazel-3.7.2.patch',
]
builddependencies = [
('binutils', '2.37'),
('Python', '3.9.6'),
('Zip', '3.0'),
]
prebuildopts = "export BAZEL_LINKOPTS=-static-libstdc++:-static-libgcc BAZEL_LINKLIBS=-l%:libstdc++.a:-lm && "
configopts = "--host_jvm_args=--illegal-access=permit"
dependencies = [('Java', '15', '', True)]
moduleclass = 'devel'
--- src/third_party/ijar/mapped_file.h.orig 2021-11-05 15:56:24.654184777 +0100
+++ src/third_party/ijar/mapped_file.h 2021-11-05 15:56:33.879015290 +0100
@@ -16,6 +16,7 @@
#define INCLUDED_THIRD_PARTY_IJAR_MAPPED_FILE_H
#include "third_party/ijar/common.h"
+#include <limits>
namespace devtools_ijar {
--- src/third_party/ijar/zlib_client.h.orig 2021-11-05 15:39:58.061336981 +0100
+++ src/third_party/ijar/zlib_client.h 2021-11-05 15:40:22.650877105 +0100
@@ -16,6 +16,7 @@
#define THIRD_PARTY_IJAR_ZLIB_CLIENT_H_
#include <limits.h>
+#include <limits>
#include "third_party/ijar/common.h"
From 0216ee54417fa1f2fef14f6eb14cbc1e8f595821 Mon Sep 17 00:00:00 2001
From: philwo <philwo@google.com>
Date: Mon, 8 Feb 2021 10:45:50 -0800
Subject: [PATCH] Fix Bazel #10214: JDK 13 introduced a source compatibility
issue.
Quote from the Java release notes:
The addition of newFileSystem(Path, Map<String, ?>) creates a source (but not binary) compatibility issue for code that has been using the existing 2-arg newFileSystem(Path, ClassLoader) and specifying the class loader as null. [...] To avoid the ambiguous reference, this code needs to be modified to cast the second parameter to java.lang.ClassLoader.
RELNOTES:
PiperOrigin-RevId: 356301318
---
.../com/google/devtools/build/buildjar/VanillaJavaBuilder.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/VanillaJavaBuilder.java b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/VanillaJavaBuilder.java
index 327017362626..5edf9ba0cf23 100644
--- a/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/VanillaJavaBuilder.java
+++ b/src/java_tools/buildjar/java/com/google/devtools/build/buildjar/VanillaJavaBuilder.java
@@ -77,7 +77,7 @@
private FileSystem getJarFileSystem(Path sourceJar) throws IOException {
FileSystem fs = filesystems.get(sourceJar);
if (fs == null) {
- filesystems.put(sourceJar, fs = FileSystems.newFileSystem(sourceJar, null));
+ filesystems.put(sourceJar, fs = FileSystems.newFileSystem(sourceJar, (ClassLoader) null));
}
return fs;
}
\ No newline at end of file
easyblock = 'ConfigureMake'
name = 'CFITSIO'
version = '4.0.0'
homepage = 'http://heasarc.gsfc.nasa.gov/fitsio/'
description = """CFITSIO is a library of C and Fortran subroutines for reading and writing data files in
FITS (Flexible Image Transport System) data format.
"""
toolchain = {'name': 'GCCcore', 'version': '11.2.0'}
toolchainopts = {'pic': True}
source_urls = ['http://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/']
sources = ['%%(namelower)s-%s.tar.gz' % version]
patches = ['CFITSIO-4.0.0_install_test_data.patch']
dependencies = [('cURL', '7.78.0')]
builddependencies = [
('binutils', '2.37'),
]
# make would create just static libcfitsio.a.
# Let's create dynamic lib and testprog too.
buildopts = '&& make shared && make testprog'
sanity_check_paths = {
'files': ['lib/libcfitsio.a', 'lib/libcfitsio.%s' % SHLIB_EXT],
'dirs': ['include'],
}
sanity_check_commands = [
('cd %(installdir)s/share && testprog'),
]
moduleclass = 'lib'
--- Makefile.in.orig 2021-11-05 17:08:01.927879000 +0100
+++ Makefile.in 2021-11-05 17:08:42.545401000 +0100
@@ -31,8 +31,9 @@
CFITSIO_BIN = ${DESTDIR}@bindir@
CFITSIO_LIB = ${DESTDIR}@libdir@
CFITSIO_INCLUDE = ${DESTDIR}@includedir@
-INSTALL_DIRS = ${DESTDIR}@INSTALL_ROOT@ ${CFITSIO_INCLUDE} ${CFITSIO_LIB} ${CFITSIO_LIB}/pkgconfig
+CFITSIO_DATADIR = ${DESTDIR}@datadir@
+INSTALL_DIRS = ${DESTDIR}@INSTALL_ROOT@ ${CFITSIO_INCLUDE} ${CFITSIO_LIB} ${CFITSIO_LIB}/pkgconfig ${CFITSIO_DATADIR}
SHELL = /bin/sh
ARCHIVE = @ARCHIVE@
easyblock = 'ConfigureMake'
name = 'libelf'
version = '0.8.13'
homepage = 'https://web.archive.org/web/20190223180146/http://www.mr511.de/software/english.html'
# The original existed here http://www.mr511.de/software/english.html'
# The only available source code is for an earlier version at https://github.com/WolfgangSt/libelf
description = """libelf is a free ELF object file access library
"""
toolchain = {'name': 'GCCcore', 'version': '11.2.0'}
source_urls = [
# source is actually a tar named as tar.gz, will fail. Download the source and gzip it
'https://web.archive.org/web/20170808201535/http://www.mr511.de/software/',
'https://fossies.org/linux/misc/old/'
]
sources = [SOURCE_TAR_GZ]
builddependencies = [
('binutils', '2.37')
]
modextrapaths = {'CPATH': 'include/libelf'}
sanity_check_paths = {
'files': ['lib/libelf.a', 'lib/libelf.%s' % SHLIB_EXT, 'lib/libelf.so.0', 'include/libelf/libelf.h'],
'dirs': ['lib/pkgconfig']
}
moduleclass = 'devel'
easyblock = 'ConfigureMake'
name = 'libgd'
version = '2.3.1'
homepage = 'https://github.com/libgd'
description = """GD is an open source code library for the dynamic creation of
images by programmers.
"""
toolchain = {'name': 'GCCcore', 'version': '11.2.0'}
source_urls = [
'https://github.com/libgd/libgd/releases/download/gd-%(version)s/']
sources = [SOURCELOWER_TAR_GZ]
builddependencies = [
('binutils', '2.37'),
]
dependencies = [
('X11', '20210802'),
('libjpeg-turbo', '2.0.6'),
('libpng', '1.6.37'),
('zlib', '1.2.11'),
]
sanity_check_paths = {
'files': ["lib/libgd.a", "lib/libgd.%s" % SHLIB_EXT],
'dirs': ["bin", "include"],
}
moduleclass = 'lib'
easyblock = 'ConfigureMake'
name = 'libgeotiff'
version = '1.7.0'
homepage = 'https://trac.osgeo.org/geotiff/'
description = """Library for reading and writing coordinate system information
from/to GeoTIFF files"""
toolchain = {'name': 'GCCcore', 'version': '11.2.0'}
source_urls = ['https://download.osgeo.org/geotiff/libgeotiff']
sources = [SOURCE_TAR_GZ]
builddependencies = [
('binutils', '2.37'),
]
dependencies = [
('PROJ', '8.1.0'),
('libjpeg-turbo', '2.0.6'),
('zlib', '1.2.11'),
('SQLite', '3.36'),
('LibTIFF', '4.3.0'),
('cURL', '7.78.0'),
]
configopts = '--with-libtiff=$EBROOTLIBTIFF'
configopts += ' --with-proj=$EBROOTPROJ '
configopts += ' --with-zlib=$EBROOTZLIB'
configopts += ' --with-jpeg=$EBROOTLIBJPEGMINTURBO'
sanity_check_paths = {
'files': ['bin/listgeo', 'lib/libgeotiff.a',
'lib/libgeotiff.%s' % SHLIB_EXT],
'dirs': ['include', 'share'],
}
moduleclass = 'lib'
easyblock = 'ConfigureMake'
name = 'libsndfile'
version = '1.0.31'
homepage = 'http://www.mega-nerd.com/libsndfile'
description = """Libsndfile is a C library for reading and writing files
containing sampled sound (such as MS Windows WAV and the Apple/SGI AIFF format)
through one standard library interface."""
toolchain = {'name': 'GCCcore', 'version': '11.2.0'}
source_urls = [
'https://github.com/libsndfile/libsndfile/releases/download/%(version)s/']
sources = [SOURCE_TAR_BZ2]
checksums = ['a8cfb1c09ea6e90eff4ca87322d4168cdbe5035cb48717b40bf77e751cc02163']
builddependencies = [('binutils', '2.37')]
configopts = '--enable-octave=no'
sanity_check_paths = {
'files': ['include/sndfile.h', 'include/sndfile.hh', 'lib/libsndfile.a',
'lib/libsndfile.%s' % SHLIB_EXT],
'dirs': ['bin'],
}
moduleclass = 'lib'
easyblock = "PythonPackage"
name = 'Mercurial'
version = '5.8'
versionsuffix = "-Python-%(pyver)s"
homepage = 'http://mercurial.selenic.com/'
description = """Mercurial is a free, distributed source control management tool. It efficiently handles projects
of any size and offers an easy and intuitive interface.
"""
toolchain = {'name': 'GCCcore', 'version': '11.2.0'}
source_urls = ['http://mercurial-scm.org/release/']
sources = [SOURCELOWER_TAR_GZ]
download_dep_fail = True
builddependencies = [('binutils', '2.37')]
dependencies = [
('Python', '3.9.6')
]
sanity_check_paths = {
'files': ['bin/hg'],
'dirs': ['lib/python%(pyshortver)s/site-packages/mercurial'],
}
moduleclass = 'tools'
##
# This file is an EasyBuild reciPY as per https://github.com/hpcugent/easybuild
#
# Copyright:: Copyright 2014-2015 The Cyprus Institute
# Authors:: Thekla Loizou <t.loizou@cyi.ac.cy>
# License:: MIT/GPL
#
##
easyblock = 'ConfigureMake'
name = 'PROJ'
version = '8.1.0'
homepage = 'http://trac.osgeo.org/proj/'
description = """Program proj is a standard Unix filter function which converts
geographic longitude and latitude coordinates into cartesian coordinates
"""
toolchain = {'name': 'GCCcore', 'version': '11.2.0'}
toolchainopts = {'pic': True}
source_urls = ['http://download.osgeo.org/proj/']
sources = [SOURCELOWER_TAR_GZ]
builddependencies = [
('binutils', '2.37'),
('SQLite', '3.36'),
('LibTIFF', '4.3.0'),
('cURL', '7.78.0'),
('pkg-config', '0.29.2'),
]
sanity_check_paths = {
'files': ['bin/cct', 'bin/cs2cs', 'bin/geod', 'bin/gie', 'bin/proj',
'bin/projinfo', 'lib/libproj.a', 'lib/libproj.%s' % SHLIB_EXT],
'dirs': ['include'],
}
moduleclass = 'lib'
#!/usr/bin/env python3
"""Try to change from GCC9 to GCC10
"""Try to change from GCC10 to GCC11
For it, you need
......@@ -7,38 +7,43 @@ For it, you need
"""
import argparse
oldGCC = '10.3.0'
newGCC = '11.2.0'
oldMKL = '2021.2.0'
newMKL = '2021.4.0'
oldgomkl = '2021'
newgomkl = '2022'
oldgpsmpi = '2021'
newgpsmpi = '2022'
oldBinutils = '2.36.1'
newBinutils = '2.37'
oldBison = '3.7.6'
newBison = '3.7.6'
oldLibDRM = '2.4.106'
newLibDRM = '2.4.107'
oldJava = '15'
newJava = '15'
oldCMake = "('CMake', '3.18.0')"
newCMake = "('CMake', '3.21.1', '', SYSTEM)"
oldCUDA = '11.3'
newCUDA = '11.5'
oldcuDNN = '8.2.1.32'
newcuDNN = '8.2.2.26'
oldScikit = '2021'
newScikit = '2022'
oldSciPy = '2021'
newSciPy = '2022'
oldprotobuf = '3.14.0'
newprotobuf = '3.17.3'
oldLibxc = '5.1.5'
newLibxc = '5.1.6'
oldPython = '3.8.5'
newPython = '3.9.6'
releases = {
'GCC': ['10.3.0', '11.2.0'],
'CMake': ['3.18.0', '31.21.1'],
'imkl': ['2021.2.0', '2021.4.0'],
'gomkl': ['2021', '2022'],
'gpsmpi': ['2021', '2022'],
'binutils': ['2.36.1', '2.37'],
'Bison': ['3.7.6', '3.7.6'],
'libdrm': ['2.4.106', '2.4.107'],
'Java': ['15', '15'],
'CUDA': ['11.3', '11.5'],
'cuDNN': ['8.2.1.32', '8.2.2.26'],
'Scikit': ['2021', '2022'],
'SciPy': ['2021', '2022'],
'protobuf': ['3.14.0', '3.17.3'],
'libxc': ['5.1.5', '5.1.6'],
'Python': ['3.8.5', '3.9.6'],
'cURL': ['7.71.1', '7.78.0'],
'PCRE': ['8.44', '8.45'],
'PCRE2': ['10.34', '10.37'],
'Qt5': ['5.14.2', '5.15.2'],
'Ninja': ['1.10.0', '1.10.2'],
're2c': ['1.3', '2.2'],
'GLib': ['2.64.4', '2.69.1'],
'X11': ['20200222', '20210802'],
'fontconfig': ['2.13.92', '2.13.94'],
'DBus': ['1.12.20', '1.13.18'],
'libjpeg-turbo': ['2.0.5', '2.0.6'],
'NSS': ['3.51', '3.69'],
'JasPer': ['2.0.19', '2.0.33'],
'snappy': ['1.1.8', '1.1.9'],
'PROJ': ['7.1.0', '8.1.0'],
'SQLite': ['3.32.3', '3.36'],
'LibTIFF': ['4.1.0', '4.3.0'],
'libgd': ['2.3.0', '2.3.1'],
}
def parse_arguments():
......@@ -50,24 +55,29 @@ def parse_arguments():
help='Current version of given software to be replaced')
parser.add_argument('-n', metavar='LATESTRELEASE', default="", type=str,
help='Most recent version of given software')
parser.add_argument('-s', metavar='SITECONTACTS', default="", type=str,
help='Update site contacts')
return parser.parse_args()
def updateFileName(fileName, current, latest):
"""Tries to bring filename automatically to the latest toolchain"""
if current and latest:
fileName = fileName.replace(current, latest, 1)
if fileName:
fileName = fileName.replace(oldGCC, newGCC, 1)
fileName = fileName.replace(oldMKL, newMKL, 1)
fileName = fileName.replace(oldgomkl, newgomkl, 1)
fileName = fileName.replace(oldgpsmpi, newgpsmpi, 1)
fileName = fileName.replace(releases['GCC'][0], releases['GCC'][1], 1)
fileName = fileName.replace(
releases['imkl'][0], releases['imkl'][1], 1)
fileName = fileName.replace(
releases['gomkl'][0], releases['gomkl'][1], 1)
fileName = fileName.replace(
releases['gpsmpi'][0], releases['gpsmpi'][1], 1)
fileName = fileName.replace(
releases['Python'][0], releases['Python'][1], 1)
return fileName
def updateFile(oldName, newName, current, latest, siteContact):
def updateFile(oldName, newName, current, latest):
"""Replaces toolchain, cleans up site_contacts and goes through the list of releases with the new one"""
if newName is not oldName:
newFile = open(newName, 'w')
with open(oldName) as file:
......@@ -78,53 +88,23 @@ def updateFile(oldName, newName, current, latest, siteContact):
line = line.replace(current, latest, 1)
if line.startswith('toolchain'):
print("Fixing toolchain...")
line = line.replace(oldGCC, newGCC, 1)
line = line.replace(oldMKL, newMKL, 1)
line = line.replace(oldgomkl, newgomkl, 1)
if siteContact:
line = line.replace(
releases['GCC'][0], releases['GCC'][1], 1)
line = line.replace(
releases['imkl'][0], releases['imkl'][1], 1)
line = line.replace(
releases['gomkl'][0], releases['gomkl'][1], 1)
line = line.replace(
releases['gpsmpi'][0], releases['gpsmpi'][1], 1)
if line.startswith('site_contacts'):
print("Replacing site contacts...")
line = "site_contacts = '%s'" % siteContact
if "binutils" in line:
print("Fixing binutils...")
line = line.replace(oldBinutils, newBinutils)
if "Bison" in line:
print("Fixing Bison...")
line = line.replace(oldBison, newBison)
if "libdrm" in line:
print("Fixing libdrm...")
line = line.replace(oldLibDRM, newLibDRM)
if "Java" in line:
print("Fixing Java...")
line = line.replace(oldJava, newJava)
if "CMake" in line:
print("Fixing Cmake...")
line = line.replace(oldCMake, newCMake)
if "cuda" in line.lower(): # There's cudaver, etc
print("Fixing CUDA...")
line = line.replace(oldCUDA, newCUDA)
if "cuDNN" in line:
print("Fixing cuDNN...")
line = line.replace(oldcuDNN, newcuDNN)
if "scikit" in line:
print("Fixing scikit...")
line = line.replace(oldScikit, newScikit)
if "gcccoremkl" in line:
print("Fixing gcccoremkl")
line = line.replace(oldGCC, newGCC)
line = line.replace(oldMKL, newMKL)
if "SciPy-Stack" in line:
print("Fixing SciPy-Stack")
line = line.replace(oldSciPy, newSciPy)
if "protobuf" in line:
print("Fixing protobuf")
line = line.replace(oldprotobuf, newprotobuf)
if "libxc" in line:
print("Fixing libxc")
line = line.replace(oldLibxc, newLibxc)
if "Python" in line:
print("Fixing Python")
line = line.replace(oldPython, newPython)
print("Removing site_contacts")
line = ""
for key in releases:
if key in line:
print("Changing", key, "from",
releases[key][0], "to", releases[key][1])
line = line.replace(
releases[key][0], releases[key][1], 1)
newFile.write(line)
newFile.close()
return True
......@@ -136,6 +116,7 @@ def updateFile(oldName, newName, current, latest, siteContact):
if __name__ == "__main__":
args = parse_arguments()
newFile = updateFileName(args.i, args.o, args.n)
if updateFile(args.i, newFile, args.o, args.n, args.s):
print("Updated", args.i, ' to GCC', newGCC, 'and to version', args.n)
if updateFile(args.i, newFile, args.o, args.n):
print("Updated", args.i, ' to GCC',
releases['GCC'][1], 'and to version', args.n)
print("Run: \n\neb -l %s" % newFile)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment