## # Copyright 2009-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), # with support of Ghent University (http://ugent.be/hpc), # the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), # Flemish Research Foundation (FWO) (http://www.fwo.be/en) # and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). # # https://github.com/easybuilders/easybuild # # EasyBuild is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation v2. # # EasyBuild is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with EasyBuild. If not, see <http://www.gnu.org/licenses/>. ## """ EasyBuild support for PETSc, implemented as an easyblock @author: Kenneth Hoste (Ghent University) """ import os import re from distutils.version import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import BUILD, CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import symlink, apply_regex_substitutions from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext from easybuild.tools.py2vs3 import string_type NO_MPI_CXX_EXT_FLAGS = '-DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX' class EB_PETSc(ConfigureMake): """Support for building and installing PETSc""" def __init__(self, *args, **kwargs): """Initialize PETSc specific variables.""" super(EB_PETSc, self).__init__(*args, **kwargs) self.petsc_arch = "" self.petsc_subdir = "" self.prefix_inc = '' self.prefix_lib = '' self.prefix_bin = '' self.with_python = False if self.cfg['sourceinstall']: self.prefix_inc = self.petsc_subdir self.prefix_lib = os.path.join(self.petsc_subdir, self.petsc_arch) self.build_in_installdir = True if LooseVersion(self.version) >= LooseVersion("3.9"): self.prefix_bin = os.path.join(self.prefix_inc, 'lib', 'petsc') @staticmethod def extra_options(): """Add extra config options specific to PETSc.""" extra_vars = { 'sourceinstall': [False, "Indicates whether a source installation should be performed", CUSTOM], 'shared_libs': [False, "Build shared libraries", CUSTOM], 'with_papi': [False, "Enable PAPI support", CUSTOM], 'papi_inc': ['/usr/include', "Path for PAPI include files", CUSTOM], 'papi_lib': ['/usr/lib64/libpapi.so', "Path for PAPI library", CUSTOM], 'runtest': ['test', "Make target to test build", BUILD], 'test_parallel': [ None, "Number of parallel PETSc tests launched. If unset, 'parallel' will be used", CUSTOM ], 'download_deps_static': [[], "Dependencies that should be downloaded and installed static", CUSTOM], 'download_deps_shared': [[], "Dependencies that should be downloaded and installed shared", CUSTOM], 'download_deps': [[], "Dependencies that should be downloaded and installed", CUSTOM] } return ConfigureMake.extra_options(extra_vars) def prepare_step(self, *args, **kwargs): """Prepare build environment.""" super(EB_PETSc, self).prepare_step(*args, **kwargs) # build with Python support if Python is loaded as a non-build (runtime) dependency build_deps = self.cfg.dependencies(build_only=True) if get_software_root('Python') and not any(x['name'] == 'Python' for x in build_deps): self.with_python = True self.log.info("Python included as runtime dependency, so enabling Python support") def configure_step(self): """ Configure PETSc by setting configure options and running configure script. Configure procedure is much more concise for older versions (< v3). """ if LooseVersion(self.version) >= LooseVersion("3"): # make the install dir first if we are doing a download install, then keep it for the rest of the way deps = self.cfg["download_deps"] + self.cfg["download_deps_static"] + self.cfg["download_deps_shared"] if deps: self.log.info("Creating the installation directory before the configure.") self.make_installdir() self.cfg["keeppreviousinstall"] = True for dep in set(deps): self.cfg.update('configopts', '--download-%s=1' % dep) for dep in self.cfg["download_deps_static"]: self.cfg.update('configopts', '--download-%s-shared=0' % dep) for dep in self.cfg["download_deps_shared"]: self.cfg.update('configopts', '--download-%s-shared=1' % dep) # compilers self.cfg.update('configopts', '--with-cc="%s"' % os.getenv('CC')) self.cfg.update('configopts', '--with-cxx="%s" --with-c++-support' % os.getenv('CXX')) self.cfg.update('configopts', '--with-fc="%s"' % os.getenv('F90')) # compiler flags # Don't build with MPI c++ bindings as this leads to a hard dependency # on libmpi and libmpi_cxx even for C code and non-MPI code cxxflags = os.getenv('CXXFLAGS') + ' ' + NO_MPI_CXX_EXT_FLAGS if LooseVersion(self.version) >= LooseVersion("3.5"): self.cfg.update('configopts', '--CFLAGS="%s"' % os.getenv('CFLAGS')) self.cfg.update('configopts', '--CXXFLAGS="%s"' % cxxflags) self.cfg.update('configopts', '--FFLAGS="%s"' % os.getenv('F90FLAGS')) else: self.cfg.update('configopts', '--with-cflags="%s"' % os.getenv('CFLAGS')) self.cfg.update('configopts', '--with-cxxflags="%s"' % cxxflags) self.cfg.update('configopts', '--with-fcflags="%s"' % os.getenv('F90FLAGS')) if not self.toolchain.comp_family() == toolchain.GCC: # @UndefinedVariable self.cfg.update('configopts', '--with-gnu-compilers=0') # MPI if self.toolchain.options.get('usempi', None): self.cfg.update('configopts', '--with-mpi=1') # build options self.cfg.update('configopts', '--with-build-step-np=%s' % self.cfg['parallel']) self.cfg.update('configopts', '--with-shared-libraries=%d' % self.cfg['shared_libs']) self.cfg.update('configopts', '--with-debugging=%d' % self.toolchain.options['debug']) self.cfg.update('configopts', '--with-pic=%d' % self.toolchain.options['pic']) self.cfg.update('configopts', '--with-x=0 --with-windows-graphics=0') # PAPI support if self.cfg['with_papi']: papi_inc = self.cfg['papi_inc'] papi_inc_file = os.path.join(papi_inc, "papi.h") papi_lib = self.cfg['papi_lib'] if os.path.isfile(papi_inc_file) and os.path.isfile(papi_lib): self.cfg.update('configopts', '--with-papi=1') self.cfg.update('configopts', '--with-papi-include=%s' % papi_inc) self.cfg.update('configopts', '--with-papi-lib=%s' % papi_lib) else: raise EasyBuildError("PAPI header (%s) and/or lib (%s) not found, can not enable PAPI support?", papi_inc_file, papi_lib) # Python extensions_step if self.with_python: # enable numpy support, but only if numpy is available (_, ec) = run_cmd("python -c 'import numpy'", log_all=True, simple=False) if ec == 0: self.cfg.update('configopts', '--with-numpy=1') # enable mpi4py support, but only if mpi4py is available (_, ec) = run_cmd("python -c 'import mpi4py'", log_all=True, simple=False) if ec == 0: with_mpi4py_opt = '--with-mpi4py' if self.cfg['shared_libs'] and with_mpi4py_opt not in self.cfg['configopts']: self.cfg.update('configopts', '%s=1' % with_mpi4py_opt) # FFTW, ScaLAPACK (and BLACS for older PETSc versions) deps = ["ScaLAPACK"] #FFTW if LooseVersion(self.version) < LooseVersion("3.5"): deps.append("BLACS") for dep in deps: libdir = os.getenv('%s_LIB_DIR' % dep.upper()) libs = os.getenv('%s_STATIC_LIBS' % dep.upper()) if libdir and libs: with_arg = "--with-%s" % dep.lower() self.cfg.update('configopts', '%s=1' % with_arg) self.cfg.update('configopts', '%s-lib=[%s/%s]' % (with_arg, libdir, libs)) inc = os.getenv('%s_INC_DIR' % dep.upper()) if inc: self.cfg.update('configopts', '%s-include=%s' % (with_arg, inc)) else: self.log.info("Missing inc/lib info, so not enabling %s support." % dep) # BLAS, LAPACK libraries bl_libdir = os.getenv('BLAS_LAPACK_LIB_DIR') bl_libs = os.getenv('BLAS_LAPACK_STATIC_LIBS') if bl_libdir and bl_libs: self.cfg.update('configopts', '--with-blas-lapack-lib=[%s/%s]' % (bl_libdir, bl_libs)) else: raise EasyBuildError("One or more environment variables for BLAS/LAPACK not defined?") # additional dependencies # filter out deps handled seperately sep_deps = ['BLACS', 'BLAS', 'CMake', 'LAPACK', 'numpy', #'FFTW' 'mpi4py', 'papi', 'ScaLAPACK', 'SciPy-bundle', 'SuiteSparse'] # SCOTCH has to be treated separately since they add weird postfixes # to library names from SCOTCH 7.0.1 or PETSc version 3.17. if (LooseVersion(self.version) >= LooseVersion("3.17")): sep_deps.append('SCOTCH') depfilter = [d['name'] for d in self.cfg.builddependencies()] + sep_deps deps = [dep['name'] for dep in self.cfg.dependencies() if not dep['name'] in depfilter] for dep in deps: if isinstance(dep, string_type): dep = (dep, dep) deproot = get_software_root(dep[0]) if deproot: if (LooseVersion(self.version) >= LooseVersion("3.5")) and (dep[1] == "SCOTCH"): withdep = "--with-pt%s" % dep[1].lower() # --with-ptscotch is the configopt PETSc >= 3.5 else: withdep = "--with-%s" % dep[1].lower() self.cfg.update('configopts', '%s=1 %s-dir=%s' % (withdep, withdep, deproot)) # SCOTCH has to be treated separately since they add weird postfixes # to library names from SCOTCH 7.0.1 or PETSc version 3.17. scotch = get_software_root('SCOTCH') scotch_ver = get_software_version('SCOTCH') if (scotch and LooseVersion(scotch_ver) >= LooseVersion("7.0")): withdep = "--with-ptscotch" scotch_inc = [os.path.join(scotch, "include")] inc_spec = "-include=[%s]" % ','.join(scotch_inc) # For some reason there is a v3 suffix added to libptscotchparmetis # which is the reason for this new code. req_scotch_libs = ['libesmumps.a', 'libptesmumps.a', 'libptscotch.a', 'libptscotcherr.a', 'libptscotchparmetisv3.a', 'libscotch.a', 'libscotcherr.a'] scotch_libs = [os.path.join(scotch, "lib", x) for x in req_scotch_libs] lib_spec = "-lib=[%s]" % ','.join(scotch_libs) self.cfg.update('configopts', ' '.join([withdep + spec for spec in ['=1', inc_spec, lib_spec]])) # SuiteSparse options changed in PETSc 3.5, suitesparse = get_software_root('SuiteSparse') if suitesparse: if LooseVersion(self.version) >= LooseVersion("3.5"): withdep = "--with-suitesparse" # specified order of libs matters! ss_libs = ["UMFPACK", "KLU", "CHOLMOD", "BTF", "CCOLAMD", "COLAMD", "CAMD", "AMD"] # More libraries added after version 3.17 if LooseVersion(self.version) >= LooseVersion("3.17"): # specified order of libs matters! ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD", "COLAMD", "CSparse", "CXSparse", "LDL", "RBio", "SLIP_LU", "CAMD", "AMD"] suitesparse_inc = [os.path.join(suitesparse, x, "Include") for x in ss_libs] suitesparse_inc.append(os.path.join(suitesparse, "SuiteSparse_config")) inc_spec = "-include=[%s]" % ','.join(suitesparse_inc) suitesparse_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.replace("_", "").lower()) for x in ss_libs] suitesparse_libs.append(os.path.join(suitesparse, "SuiteSparse_config", "libsuitesparseconfig.a")) lib_spec = "-lib=[%s]" % ','.join(suitesparse_libs) else: # CHOLMOD and UMFPACK are part of SuiteSparse (PETSc < 3.5) withdep = "--with-umfpack" inc_spec = "-include=%s" % os.path.join(suitesparse, "UMFPACK", "Include") # specified order of libs matters! umfpack_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.lower()) for x in ["UMFPACK", "CHOLMOD", "COLAMD", "AMD"]] lib_spec = "-lib=[%s]" % ','.join(umfpack_libs) self.cfg.update('configopts', ' '.join([withdep + spec for spec in ['=1', inc_spec, lib_spec]])) # set PETSC_DIR for configure (env) and build_step env.setvar('PETSC_DIR', self.cfg['start_dir']) self.cfg.update('buildopts', 'PETSC_DIR=%s' % self.cfg['start_dir']) if self.cfg['sourceinstall']: # run configure without --prefix (required) cmd = "%s ./configure %s" % (self.cfg['preconfigopts'], self.cfg['configopts']) (out, _) = run_cmd(cmd, log_all=True, simple=False) else: out = super(EB_PETSc, self).configure_step() # check for errors in configure error_regexp = re.compile("ERROR") if error_regexp.search(out): raise EasyBuildError("Error(s) detected in configure output!") if self.cfg['sourceinstall']: # figure out PETSC_ARCH setting petsc_arch_regex = re.compile(r"^\s*PETSC_ARCH:\s*(\S+)$", re.M) res = petsc_arch_regex.search(out) if res: self.petsc_arch = res.group(1) self.cfg.update('buildopts', 'PETSC_ARCH=%s' % self.petsc_arch) else: raise EasyBuildError("Failed to determine PETSC_ARCH setting.") self.petsc_subdir = '%s-%s' % (self.name.lower(), self.version) else: # old versions (< 3.x) self.cfg.update('configopts', '--prefix=%s' % self.installdir) self.cfg.update('configopts', '--with-shared=1') # additional dependencies for dep in ["SCOTCH"]: deproot = get_software_root(dep) if deproot: withdep = "--with-%s" % dep.lower() self.cfg.update('configopts', '%s=1 %s-dir=%s' % (withdep, withdep, deproot)) cmd = "./config/configure.py %s" % self.get_cfg('configopts') run_cmd(cmd, log_all=True, simple=True) # Make sure to set test_parallel before self.cfg['parallel'] is set to None if self.cfg['test_parallel'] is None and self.cfg['parallel']: self.cfg['test_parallel'] = self.cfg['parallel'] # PETSc > 3.5, make does not accept -j # to control parallel build, we need to specify MAKE_NP=... as argument to 'make' command if LooseVersion(self.version) >= LooseVersion("3.5"): self.cfg.update('buildopts', "MAKE_NP=%s" % self.cfg['parallel']) self.cfg['parallel'] = None # default make should be fine def test_step(self): """ Test the compilation """ # Each PETSc test may use multiple threads, so running "self.cfg['parallel']" of them may lead to # some oversubscription every now and again. Not a big deal, but if needed a reduced parallelism # can be specified with test_parallel - and it takes priority paracmd = '' self.log.info("In test_step: %s" % self.cfg['test_parallel']) if self.cfg['test_parallel'] is not None: paracmd = "-j %s" % self.cfg['test_parallel'] if self.cfg['runtest']: cmd = "%s make %s %s %s" % (self.cfg['pretestopts'], paracmd, self.cfg['runtest'], self.cfg['testopts']) (out, _) = run_cmd(cmd, log_all=True, simple=False) return out def install_step(self): """ Install using make install (for non-source installations), or by symlinking files (old versions, < 3). """ if LooseVersion(self.version) >= LooseVersion("3"): if not self.cfg['sourceinstall']: super(EB_PETSc, self).install_step() petsc_root = self.installdir else: petsc_root = os.path.join(self.installdir, self.petsc_subdir) # Remove MPI-CXX flags added during configure to prevent them from being passed to consumers of PETsc petsc_variables_path = os.path.join(petsc_root, 'lib', 'petsc', 'conf', 'petscvariables') if os.path.isfile(petsc_variables_path): fix = (r'^(CXX_FLAGS|CXX_LINKER_FLAGS|CONFIGURE_OPTIONS)( = .*)%s(.*)$' % NO_MPI_CXX_EXT_FLAGS, r'\1\2\3') apply_regex_substitutions(petsc_variables_path, [fix]) else: # old versions (< 3.x) for fn in ['petscconf.h', 'petscconfiginfo.h', 'petscfix.h', 'petscmachineinfo.h']: includedir = os.path.join(self.installdir, 'include') bmakedir = os.path.join(self.installdir, 'bmake', 'linux-gnu-c-opt') symlink(os.path.join(bmakedir, fn), os.path.join(includedir, fn)) def make_module_req_guess(self): """Specify PETSc custom values for PATH, CPATH and LD_LIBRARY_PATH.""" guesses = super(EB_PETSc, self).make_module_req_guess() guesses.update({ 'CPATH': [os.path.join(self.prefix_lib, 'include'), os.path.join(self.prefix_inc, 'include')], 'LD_LIBRARY_PATH': [os.path.join(self.prefix_lib, 'lib')], 'PATH': [os.path.join(self.prefix_bin, 'bin')], # see https://www.mcs.anl.gov/petsc/documentation/faq.html#sparse-matrix-ascii-format 'PYTHONPATH': [os.path.join('lib', 'petsc', 'bin')], }) return guesses def make_module_extra(self): """Set PETSc specific environment variables (PETSC_DIR, PETSC_ARCH).""" txt = super(EB_PETSc, self).make_module_extra() if self.cfg['sourceinstall']: txt += self.module_generator.set_environment('PETSC_DIR', os.path.join(self.installdir, self.petsc_subdir)) txt += self.module_generator.set_environment('PETSC_ARCH', self.petsc_arch) else: txt += self.module_generator.set_environment('PETSC_DIR', self.installdir) return txt def sanity_check_step(self): """Custom sanity check for PETSc""" if self.cfg['shared_libs']: libext = get_shared_lib_ext() else: libext = 'a' custom_paths = { 'files': [os.path.join(self.prefix_lib, 'lib', 'libpetsc.%s' % libext)], 'dirs': [os.path.join(self.prefix_bin, 'bin'), os.path.join(self.prefix_inc, 'include'), os.path.join(self.prefix_lib, 'include')] } if LooseVersion(self.version) < LooseVersion('3.6'): custom_paths['dirs'].append(os.path.join(self.prefix_lib, 'conf')) else: custom_paths['dirs'].append(os.path.join(self.prefix_lib, 'lib', 'petsc', 'conf')) custom_commands = [] if self.with_python: custom_commands.append("python -m PetscBinaryIO --help") super(EB_PETSc, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)