Commit 6cf2118f authored by Stepan Nassyr's avatar Stepan Nassyr
Browse files

Add CANN-Toolkit 5.0.2.alpha005 and it's dependencies

parent f8d0d5db
##
# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild
#
# Copyright:: Copyright 2022 Forschungszentrum Juelich GmbH
# Authors:: Stepan Nassyr
# License:: MIT/GPL
# $Id$
##
"""
EasyBuild support for Huawei CANN-related packages, implemented as an easyblock
@author: Stepan Nassyr (Forschungszentrum Juelich)
"""
import os
import stat
import platform
import grp
from distutils.version import LooseVersion
from easybuild.easyblocks.generic.binary import Binary
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import apply_regex_substitutions,adjust_permissions,mkdir,symlink,remove_file
from easybuild.tools.run import run_cmd
class cann_package(Binary):
"""
Support for installing CANN packages.
"""
@staticmethod
def extra_options():
"""Support for generic 'default' modules with specific real versions"""
extra_vars = {
'cann_installers': [None, "List of tuples: name, path to installation script (after extraction) and additional arguments for each package, i.e [(Ascend-acllib, acllib/scripts/install.sh, ''), ...]", CUSTOM],
'subdir_symlinks': [None, "List of tuples: symlinks to create, alt_suf will be replaced, i.e ('opp','opp_linux.alt_suf') would create opp_linux.arm64 -> arm64-linux/opp on an aarch64 system", CUSTOM]
}
return extra_vars
subinstallers = {}
package_dirs = {}
def fix_scripts(self, package_dir):
# get all shell scripts in package directory
scripts = []
for root,dirs,files in os.walk(package_dir, topdown=False):
for name in files:
if name.endswith('.sh'):
scripts.append(os.path.join(root,name))
subs = [
(r'`cat /etc/passwd \| cut -f1 -d\':\' \| grep -w "(\$[^"]+)" -c`',r'$(getent passwd \1 | grep -w "\1" -c)'),
(r'^(un|)chattrFiles\(\)\s*{',r'\1chattrFiles() {'+'\n'+'return'+'\n'+'}'+'\n'+r'\1chattrFiles_disabled() {'),
(r'^(function\s*|)checkEmptyUserAndGroup\(\)\s*{',r'\1checkEmptyUserAndGroup() {'+'\n'+'return'+'\n'+'}'+'\n'+r'\1checkEmptyUserAndGroup_disabled() {'),
(r'^checkInstallCondition\(\)\s*{',r'checkInstallCondition() {'+'\n'+'return'+'\n'+'}'+'\n'+r'checkInstallCondition_disabled() {'),
]
apply_regex_substitutions(scripts, subs)
def extract_step(self):
"""Extract main installer and any subpackages inside it, then fix the """
cann_installers = self.cfg['cann_installers']
main_installer = self.src[0]['path']
(out,ret) = run_cmd("/bin/sh " + main_installer + " --noexec --extract=" + self.builddir)
if 0 != ret:
raise EasyBuildError("Main installer " + main_installer + " failed. output: " + out)
# Gather the subinstallers
subinst_dir = os.path.join(self.builddir,'run_package')
for root,dirs,files in os.walk(subinst_dir, topdown=False):
for name in files:
if name.endswith('.run'):
for package in cann_installers:
package_name = package[0]
if name.startswith(package_name):
self.subinstallers[package_name] = os.path.join(root,name)
self.package_dirs[package_name] = (os.path.join(root,name.replace('.run','-unpacked')))
# extract the subinstallers
for package in cann_installers:
package_name = package[0]
(out,ret) = run_cmd("/bin/sh " + self.subinstallers[package_name] + " --noexec --extract=" + self.package_dirs[package_name])
if 0 != ret:
raise EasyBuildError("Subinstaller " + self.subinstallers[package_name] + " failed. output: " + out)
for package in cann_installers:
package_name = package[0]
self.fix_scripts(self.package_dirs[package_name])
def install_step(self):
""" Run all subinstallers """
hiai_group = os.getenv("HIAIGROUP","InvalidGroup")
if "InvalidGroup" == hiai_group:
raise EasyBuildError("Please specify the group for using Ascend devices in the HIAIGROUP environment variable")
arch_suf = platform.machine()
alt_suf = {
'x86_64' : 'x64',
'aarch64' : 'arm64'
}
alt_suf = alt_suf[arch_suf]
installdir_toolkit=os.path.join(self.installdir,'ascend-toolkit',self.version,alt_suf+'-linux')
mkdir(installdir_toolkit,parents=True)
cann_installers = self.cfg['cann_installers']
parent_dir = os.getcwd()
common_args = " --quiet --full --install-username=$USER --install-usergroup=" + hiai_group + " --install-path=" + installdir_toolkit + ' '
for package in cann_installers:
package_name = package[0]
installer_path = package[1]
custom_args = package[2]
os.chdir(self.package_dirs[package_name])
(out,ret) = run_cmd("/bin/sh " + installer_path + " --placeholder --placeholder " + custom_args + ' ' + common_args)
if 0 != ret:
raise EasyBuildError("Installing " + package_name + " failed: " + out)
os.chdir(parent_dir)
def post_install_step(self):
"""chmod + chown everything so it's usable and create symlinks"""
hiai_group = os.getenv("HIAIGROUP","InvalidGroup")
if "InvalidGroup" == hiai_group:
raise EasyBuildError("Please specify the group for using Ascend devices in the HIAIGROUP environment variable")
hiai_gid = grp.getgrnam(hiai_group).gr_gid
adjust_permissions(self.installdir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP, add=True, recursive=True, relative=True, onlydirs=True, group_id=hiai_gid)
adjust_permissions(self.installdir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP, add=True, recursive=True, relative=True, onlyfiles=True, group_id=hiai_gid)
arch_suf = platform.machine()
alt_suf = {
'x86_64' : 'x64',
'aarch64' : 'arm64'
}
alt_suf = alt_suf[arch_suf]
installdir_toolkit=os.path.join(self.installdir,'ascend-toolkit',self.version)
prev_dir = os.getcwd()
os.chdir(installdir_toolkit)
subdir_symlinks = self.cfg['subdir_symlinks']
for sl in subdir_symlinks:
src = os.path.join(alt_suf+'-linux',sl[0])
dst = sl[1].replace('alt_suf',alt_suf)
if os.path.islink(dst):
remove_file(dst)
symlink(src,dst)
os.chdir(prev_dir)
super(cann_package, self).post_install_step()
# Stepan Nassyr (JSC)
#easyblock = 'Binary'
easyblock = 'cann_package'
name = 'CANN-Toolkit'
version = '5.0.2.alpha005'
homepage = 'https://e.huawei.com/en/products/cloud-computing-dc/atlas/cann'
description = 'CANN (Compute Architecture for Neural Networks) is the development kit for Huaweis Atlas/Ascend AI/ML products'
versionsuffix = '-Python-%(pyver)s'
toolchain = {'name': 'goolf', 'version': '2021a.9'}
import platform as local_platform
local_arch_suf = local_platform.machine()
local_alt_suf = {
'x86_64' : 'x64',
'aarch64' : 'arm64'
}
local_arch_sums = {
'x86_64' : '916f3aea0fbad4d22bc9fe6ac2f19f8f17a73812cbc2f557e50d8b9f32d140f4',
'aarch64' : 'b51ea38e869663a6fa46c879691480c0997098fea78d3c4ac2b9f64d9dcf2ead',
}
sources = [{
'source_urls' : ['https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/%(version)s/'],
'filename': 'Ascend-cann-toolkit_%%(version)s_linux-%s.run' % local_arch_suf,
}]
patches = [('cann-toolkit-getuserinfo.patch','run_package')]
checksums = [
local_arch_sums[local_arch_suf],
'9c6e7cd4e543b0b23de1472a63c8a1b5b83f2ff60918a313ffe526c1131b71ff',
]
dependencies = [
('CMake', '3.20.0'),
('Python', '3.7.5'),
('SciPy-Stack', '2021a', versionsuffix),
]
cann_installers=[
('Ascend-acllib','acllib/scripts/install.sh',''),
('Ascend-pyACL','script/install.sh',''),
('Ascend-atc','atc/scripts/install.sh','--pylocal'),
('Ascend-opp','opp/script/install.sh',''),
('Ascend-toolkit','toolkit/scripts/install.sh','--pylocal'),
('Ascend-fwkacllib','fwkacllib/scripts/install.sh','--pylocal'),
('Ascend310-aicpu_kernels','aicpu/script/install.sh',''),
('Ascend910-aicpu_kernels','aicpu/script/install.sh',''),
('Ascend310-aicpu_kernels_minirc','aicpu/script/install.sh',''),
]
subdir_symlinks=[
('acllib','acllib'),
('acllib','acllib_linux.alt_suf'),
('atc','atc'),
('fwkacllib','fwkacllib'),
('opp','opp'),
('opp','opp_linux.alt_suf'),
('pyACL','pyACL'),
('python','python'),
('targets','targets'),
('toolkit','toolkit'),
('toolkit','toolkit'),
('tools','tools'),
('var','var'),
]
sanity_check_paths = {
'files': ['ascend-toolkit/%(version)s/atc/bin/atc'],
'dirs' :['ascend-toolkit/%%(version)s/%s' % d for d in ['acllib','acllib_linux.%s' %local_alt_suf[local_arch_suf],\
'%s-linux' % local_alt_suf[local_arch_suf],\
'atc','fwkacllib',\
'opp','opp_linux.%s' % local_alt_suf[local_arch_suf],\
'pyACL','python','targets','toolkit','tools','var']] + \
['ascend-toolkit/%%(version)s/atc/%s' % d for d in ['include','lib64','bin']] + \
['ascend-toolkit/%%(version)s/acllib/%s' % d for d in ['include','lib64','bin']] + \
['ascend-toolkit/%%(version)s/fwkacllib/%s' % d for d in ['lib64','bin']] + \
['ascend-toolkit/%%(version)s/%s/python/site-packages' % d for d in ['fwkacllib','toolkit','atc']] + \
['ascend-toolkit/%(version)s/pyACL/python/site-packages/acl']
}
modextrapaths = {
'PATH' : ['ascend-toolkit/%(version)s/fwkacllib/ccec_compiler/bin',
'ascend-toolkit/%(version)s/fwkacllib/bin',
'ascend-toolkit/%(version)s/atc/bin'],
'PYTHONPATH' : ['ascend-toolkit/%(version)s/fwkacllib/python/site-packages',
'ascend-toolkit/%(version)s/toolkit/python/site-packages',
'ascend-toolkit/%(version)s/atc/python/site-packages',
'ascend-toolkit/%(version)s/pyACL/python/site-packages/acl'],
'LD_LIBRARY_PATH' : ['ascend-toolkit/%(version)s/fwkacllib/lib64','ascend-toolkit/%(version)s/atc/lib64'],
'LIBRARY_PATH' : ['ascend-toolkit/%(version)s/fwkacllib/lib64','ascend-toolkit/%(version)s/atc/lib64'],
'CPATH' : 'ascend-toolkit/%(version)s/acllib/include'
}
modextravars = {
'ASCEND_AICPU_PATH' : '%(installdir)s/ascend-toolkit/%(version)s',
'ASCEND_OPP_PATH' : '%(installdir)s/ascend-toolkit/%(version)s/opp',
'TOOLCHAIN_HOME' : '%(installdir)s/ascend-toolkit/%(version)s/toolkit',
}
moduleclass = 'lib'
diff --color -urN Ascend-toolkit-1.78.t30.0.b300-linux.aarch64-unpatched/toolkit/scripts/install.sh Ascend-toolkit-1.78.t30.0.b300-linux.aarch64-unpacked/toolkit/scripts/install.sh
--- Ascend-toolkit-1.78.t30.0.b300-linux.aarch64-unpatched/toolkit/scripts/install.sh 2022-01-22 12:42:47.889225179 +0000
+++ Ascend-toolkit-1.78.t30.0.b300-linux.aarch64-unpacked/toolkit/scripts/install.sh 2022-01-22 23:00:50.213937140 +0000
@@ -180,39 +180,39 @@
}
getUserInfo() {
- if [ "$(id -u)" -ne 0 ];then
- # common user
- if [ $input_username_flag = y ] || [ $input_usergroup_flag = y ]; then
- log_and_print $LEVEL_ERROR "No need to specify the username and usergroup."
- exitLog 1
- fi
- username=$USER # current username
- local usergroups=`groups`
- usergroup=${usergroups%%" "*}
- else
- # root
- if [ $input_username_flag = y ]; then
- if [ ! $input_usergroup_flag = y ]; then
- log_and_print $LEVEL_ERROR "Username and usergroup must be specified at the same time."
- exitLog 1
- fi
- else
- if [ $input_usergroup_flag = y ]; then
- log_and_print $LEVEL_ERROR "Username and usergroup must be specified at the same time."
- exitLog 1
- fi
- # use default value
- username=$(getInstallParam "UserName" "${installInfo}")
- usergroup=$(getInstallParam "UserGroup" "${installInfo}")
- if [ x"${username}" == "x" ]; then
- username=$DEFAULT_USERNAME
- fi
- if [ x"${usergroup}" == "x" ]; then
- usergroup=$DEFAULT_USERGROUP
- fi
- return
- fi
- fi
+ #if [ "$(id -u)" -ne 0 ];then
+ # # common user
+ # if [ $input_username_flag = y ] || [ $input_usergroup_flag = y ]; then
+ # log_and_print $LEVEL_ERROR "No need to specify the username and usergroup."
+ # exitLog 1
+ # fi
+ # username=$USER # current username
+ # local usergroups=`groups`
+ # usergroup=${usergroups%%" "*}
+ #else
+ # # root
+ # if [ $input_username_flag = y ]; then
+ # if [ ! $input_usergroup_flag = y ]; then
+ # log_and_print $LEVEL_ERROR "Username and usergroup must be specified at the same time."
+ # exitLog 1
+ # fi
+ # else
+ # if [ $input_usergroup_flag = y ]; then
+ # log_and_print $LEVEL_ERROR "Username and usergroup must be specified at the same time."
+ # exitLog 1
+ # fi
+ # # use default value
+ # username=$(getInstallParam "UserName" "${installInfo}")
+ # usergroup=$(getInstallParam "UserGroup" "${installInfo}")
+ # if [ x"${username}" == "x" ]; then
+ # username=$DEFAULT_USERNAME
+ # fi
+ # if [ x"${usergroup}" == "x" ]; then
+ # usergroup=$DEFAULT_USERGROUP
+ # fi
+ # return
+ # fi
+ #fi
if [ ! -f "${installInfo}" ]; then
return
fi
# This file is part of the AArch64/Prototype JSC EasyBuild repository (https://gitlab.version.fz-juelich.de/nassyr1/juawei-easyconfigs/)
easyblock = 'CMakeNinja'
name = 'gRPC'
version = '1.37.1'
homepage = 'https://grpc.io/'
description = """gRPC is a modern open source high performance Remote Procedure Call (RPC) framework that can run in any environment.
"""
toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
github_account = 'grpc'
source_urls = [GITHUB_LOWER_SOURCE]
sources = ['v%(version)s.tar.gz']
builddependencies = [
('CMake', '3.20.0'),
('Ninja', '1.10.2'),
('binutils', '2.36.1'),
('Autotools', '20210330'),
('libtool', '2.4.6'),
('pkg-config', '0.29.2'),
]
dependencies = [
('protobuf', '3.15.6'),
('abseil-cpp', '20210324.1'),
('c-ares', '1.17.1'),
('re2', '2021-04-01'),
('zlib', '1.2.11'),
]
configopts = "-DgRPC_CARES_PROVIDER=package -DgRPC_ABSL_PROVIDER=package -DgRPC_PROTOBUF_PROVIDER=package -DgRPC_ZLIB_PROVIDER=package -DgRPC_RE2_PROVIDER=package -DgRPC_SSL_PROVIDER=package "
sanity_check_paths = {
'files': ['include/grpc/%s.h' % x for x in ['grpc','grpc_posix','grpc_security','grpc_security_constants']]+
['include/grpc++/%s.h' % x for x in ['grpc++','server','server_posix']]+
['include/grpcpp/%s.h' % x for x in ['grpcpp','server','server_posix']]+
['lib/libgrpc%s.a' % x for x in ['','++','++_alts','++_error_details','_plugin_support','pp_channelz','++_reflection','++_unsecure','_unsecure']]+
['lib/cmake/grpc/gRPC%s.cmake' % x for x in ['Config','ConfigVersion','Targets','Targets-release']]+
['lib/pkgconfig/%s.pc' % x for x in ['gpr','grpc++','grpc','grpc++_unsecure','grpc_unsecure']],
'dirs': [],
}
moduleclass = 'tools'
easyblock = "PythonPackage"
name = 'h5py'
version = '3.5.0'
versionsuffix = '-serial'
homepage = 'http://www.h5py.org/'
description = """HDF5 for Python (h5py) is a general-purpose Python interface to the Hierarchical Data Format library,
version 5. HDF5 is a versatile, mature scientific software library designed for the fast, flexible storage of enormous
amounts of data.
"""
toolchain = {'name': 'GCCcore', 'version': '10.3.0'}
sources = [SOURCE_TAR_GZ]
checksums = ['77c7be4001ac7d3ed80477de5b6942501d782de1bbe4886597bdfec2a7ab821f']
builddependencies = [
('pkgconfig', '1.5.2', '-Python-3.9.4'),
('binutils', '2.36.1'),
]
dependencies = [
('Python', '3.9.4'),
('SciPy-Stack', '2021a', '-Python-3.9.4', ('goolf', '2021a.10')),
('HDF5', '1.12.0', '-serial'),
]
use_pip = True
sanity_pip_check = True
download_dep_fail = True
# h5py's setup.py will disable setup_requires if H5PY_SETUP_REQUIRES is set to 0
# without this environment variable, pip will fetch the minimum numpy version h5py supports during install,
# even though SciPy-bundle provides a newer version that satisfies h5py's install_requires dependency.
preinstallopts = 'HDF5_DIR="$EBROOTHDF5" H5PY_SETUP_REQUIRES=0 '
moduleclass = 'data'
easyblock = "PythonPackage"
name = 'h5py'
version = '3.5.0'
versionsuffix = '-serial'
homepage = 'http://www.h5py.org/'
description = """HDF5 for Python (h5py) is a general-purpose Python interface to the Hierarchical Data Format library,
version 5. HDF5 is a versatile, mature scientific software library designed for the fast, flexible storage of enormous
amounts of data.
"""
toolchain = {'name': 'GCCcore', 'version': '11.1.0'}
sources = [SOURCE_TAR_GZ]
checksums = ['77c7be4001ac7d3ed80477de5b6942501d782de1bbe4886597bdfec2a7ab821f']
builddependencies = [
('pkgconfig', '1.5.2', '-Python-3.9.4'),
('binutils', '2.36.1'),
]
dependencies = [
('Python', '3.9.4'),
('SciPy-Stack', '2021a', '-Python-3.9.4', ('goolf', '2021a.11')),
('HDF5', '1.12.0', '-serial'),
]
use_pip = True
sanity_pip_check = True
download_dep_fail = True
# h5py's setup.py will disable setup_requires if H5PY_SETUP_REQUIRES is set to 0
# without this environment variable, pip will fetch the minimum numpy version h5py supports during install,
# even though SciPy-bundle provides a newer version that satisfies h5py's install_requires dependency.
preinstallopts = 'HDF5_DIR="$EBROOTHDF5" H5PY_SETUP_REQUIRES=0 '
moduleclass = 'data'
easyblock = "PythonPackage"
name = 'h5py'
version = '3.5.0'
versionsuffix = '-serial'
homepage = 'http://www.h5py.org/'
description = """HDF5 for Python (h5py) is a general-purpose Python interface to the Hierarchical Data Format library,
version 5. HDF5 is a versatile, mature scientific software library designed for the fast, flexible storage of enormous
amounts of data.
"""
toolchain = {'name': 'GCCcore', 'version': '9.3.0'}
sources = [SOURCE_TAR_GZ]
checksums = ['77c7be4001ac7d3ed80477de5b6942501d782de1bbe4886597bdfec2a7ab821f']
builddependencies = [
('pkgconfig', '1.5.2', '-Python-3.9.4'),
('binutils', '2.36.1'),
]
dependencies = [
('Python', '3.9.4'),
('SciPy-Stack', '2021a', '-Python-3.9.4', ('goolf', '2021a.9')),
('HDF5', '1.12.0', '-serial'),
]
use_pip = True
sanity_pip_check = True
download_dep_fail = True
# h5py's setup.py will disable setup_requires if H5PY_SETUP_REQUIRES is set to 0
# without this environment variable, pip will fetch the minimum numpy version h5py supports during install,
# even though SciPy-bundle provides a newer version that satisfies h5py's install_requires dependency.
preinstallopts = 'HDF5_DIR="$EBROOTHDF5" H5PY_SETUP_REQUIRES=0 '
moduleclass = 'data'
easyblock = "PythonPackage"
name = 'h5py'
version = '3.5.0'
versionsuffix = '-Python-%(pyver)s'
homepage = 'http://www.h5py.org/'
description = """HDF5 for Python (h5py) is a general-purpose Python interface to the Hierarchical Data Format library,
version 5. HDF5 is a versatile, mature scientific software library designed for the fast, flexible storage of enormous
amounts of data.
"""
toolchain = {'name': 'goolf', 'version': '2021a.9'}
sources = [SOURCE_TAR_GZ]
checksums = ['77c7be4001ac7d3ed80477de5b6942501d782de1bbe4886597bdfec2a7ab821f']
builddependencies = [
('pkgconfig', '1.5.2', '-Python-3.7.5'),
('binutils', '2.36.1'),
]
dependencies = [
('Python', '3.7.5'),
('SciPy-Stack', '2021a', '-Python-3.7.5'),
('HDF5', '1.12.0'),
]
use_pip = True
sanity_pip_check = True
download_dep_fail = True
# h5py's setup.py will disable setup_requires if H5PY_SETUP_REQUIRES is set to 0
# without this environment variable, pip will fetch the minimum numpy version h5py supports during install,
# even though SciPy-bundle provides a newer version that satisfies h5py's install_requires dependency.
preinstallopts = 'HDF5_DIR="$EBROOTHDF5" H5PY_SETUP_REQUIRES=0 '
moduleclass = 'data'
easyblock = "PythonPackage"
name = 'h5py'
version = '3.5.0'
homepage = 'http://www.h5py.org/'
description = """HDF5 for Python (h5py) is a general-purpose Python interface to the Hierarchical Data Format library,
version 5. HDF5 is a versatile, mature scientific software library designed for the fast, flexible storage of enormous
amounts of data.
"""
toolchain = {'name': 'goolf', 'version': '2021a.9'}
sources = [SOURCE_TAR_GZ]
checksums = ['77c7be4001ac7d3ed80477de5b6942501d782de1bbe4886597bdfec2a7ab821f']
builddependencies = [
('pkgconfig', '1.5.2', '-Python-3.9.4'),
('binutils', '2.36.1'),
]
dependencies = [
('Python', '3.9.4'),
('SciPy-Stack', '2021a', '-Python-3.9.4'),
('HDF5', '1.12.0'),
]
use_pip = True
sanity_pip_check = True
download_dep_fail = True
# h5py's setup.py will disable setup_requires if H5PY_SETUP_REQUIRES is set to 0
# without this environment variable, pip will fetch the minimum numpy version h5py supports during install,
# even though SciPy-bundle provides a newer version that satisfies h5py's install_requires dependency.
preinstallopts = 'HDF5_DIR="$EBROOTHDF5" H5PY_SETUP_REQUIRES=0 '
moduleclass = 'data'
This diff is collapsed.
# This file is part of JSC's public easybuild repository (https://github.com/easybuilders/jsc)
easyblock = 'PythonPackage'
name = 'pkgconfig'