finally a working petsc version, links partly hardcoded, I may need...

finally a working petsc version, links partly hardcoded, I may need improvement here- alternatively, links could be set within imkl

Custom_EasyBlocks/petsc.py

+##
+# Copyright 2009-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for PETSc, implemented as an easyblock
+
+@author: Kenneth Hoste (Ghent University)
+"""
+import os
+import re
+from distutils.version import LooseVersion
+
+import easybuild.tools.environment as env
+import easybuild.tools.toolchain as toolchain
+from easybuild.easyblocks.generic.configuremake import ConfigureMake
+from easybuild.framework.easyconfig import BUILD, CUSTOM
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.filetools import symlink, apply_regex_substitutions
+from easybuild.tools.modules import get_software_root, get_software_version
+from easybuild.tools.run import run_cmd
+from easybuild.tools.systemtools import get_shared_lib_ext
+from easybuild.tools.py2vs3 import string_type
+
+NO_MPI_CXX_EXT_FLAGS = '-DOMPI_SKIP_MPICXX -DMPICH_SKIP_MPICXX'
+
+
+class EB_PETSc(ConfigureMake):
+    """Support for building and installing PETSc"""
+
+    def __init__(self, *args, **kwargs):
+        """Initialize PETSc specific variables."""
+        super(EB_PETSc, self).__init__(*args, **kwargs)
+
+        self.petsc_arch = ""
+        self.petsc_subdir = ""
+        self.prefix_inc = ''
+        self.prefix_lib = ''
+        self.prefix_bin = ''
+
+        self.with_python = False
+
+        if self.cfg['sourceinstall']:
+            self.prefix_inc = self.petsc_subdir
+            self.prefix_lib = os.path.join(self.petsc_subdir, self.petsc_arch)
+            self.build_in_installdir = True
+
+        if LooseVersion(self.version) >= LooseVersion("3.9"):
+            self.prefix_bin = os.path.join(self.prefix_inc, 'lib', 'petsc')
+
+    @staticmethod
+    def extra_options():
+        """Add extra config options specific to PETSc."""
+        extra_vars = {
+            'sourceinstall': [False, "Indicates whether a source installation should be performed", CUSTOM],
+            'shared_libs': [False, "Build shared libraries", CUSTOM],
+            'with_papi': [False, "Enable PAPI support", CUSTOM],
+            'papi_inc': ['/usr/include', "Path for PAPI include files", CUSTOM],
+            'papi_lib': ['/usr/lib64/libpapi.so', "Path for PAPI library", CUSTOM],
+            'runtest': ['test', "Make target to test build", BUILD],
+            'test_parallel': [
+                None,
+                "Number of parallel PETSc tests launched. If unset, 'parallel' will be used",
+                CUSTOM
+            ],
+            'download_deps_static': [[], "Dependencies that should be downloaded and installed static", CUSTOM],
+            'download_deps_shared': [[], "Dependencies that should be downloaded and installed shared", CUSTOM],
+            'download_deps': [[], "Dependencies that should be downloaded and installed", CUSTOM]
+        }
+        return ConfigureMake.extra_options(extra_vars)
+
+    def prepare_step(self, *args, **kwargs):
+        """Prepare build environment."""
+
+        super(EB_PETSc, self).prepare_step(*args, **kwargs)
+
+        # build with Python support if Python is loaded as a non-build (runtime) dependency
+        build_deps = self.cfg.dependencies(build_only=True)
+        if get_software_root('Python') and not any(x['name'] == 'Python' for x in build_deps):
+            self.with_python = True
+            self.log.info("Python included as runtime dependency, so enabling Python support")
+
+    def configure_step(self):
+        """
+        Configure PETSc by setting configure options and running configure script.
+
+        Configure procedure is much more concise for older versions (< v3).
+        """
+        if LooseVersion(self.version) >= LooseVersion("3"):
+            # make the install dir first if we are doing a download install, then keep it for the rest of the way
+            deps = self.cfg["download_deps"] + self.cfg["download_deps_static"] + self.cfg["download_deps_shared"]
+            if deps:
+                self.log.info("Creating the installation directory before the configure.")
+                self.make_installdir()
+                self.cfg["keeppreviousinstall"] = True
+                for dep in set(deps):
+                    self.cfg.update('configopts', '--download-%s=1' % dep)
+                for dep in self.cfg["download_deps_static"]:
+                    self.cfg.update('configopts', '--download-%s-shared=0' % dep)
+                for dep in self.cfg["download_deps_shared"]:
+                    self.cfg.update('configopts', '--download-%s-shared=1' % dep)
+
+            # compilers
+            self.cfg.update('configopts', '--with-cc="%s"' % os.getenv('CC'))
+            self.cfg.update('configopts', '--with-cxx="%s" --with-c++-support' % os.getenv('CXX'))
+            self.cfg.update('configopts', '--with-fc="%s"' % os.getenv('F90'))
+
+            # compiler flags
+            # Don't build with MPI c++ bindings as this leads to a hard dependency
+            # on libmpi and libmpi_cxx even for C code and non-MPI code
+            cxxflags = os.getenv('CXXFLAGS') + ' ' + NO_MPI_CXX_EXT_FLAGS
+            if LooseVersion(self.version) >= LooseVersion("3.5"):
+                self.cfg.update('configopts', '--CFLAGS="%s"' % os.getenv('CFLAGS'))
+                self.cfg.update('configopts', '--CXXFLAGS="%s"' % cxxflags)
+                self.cfg.update('configopts', '--FFLAGS="%s"' % os.getenv('F90FLAGS'))
+            else:
+                self.cfg.update('configopts', '--with-cflags="%s"' % os.getenv('CFLAGS'))
+                self.cfg.update('configopts', '--with-cxxflags="%s"' % cxxflags)
+                self.cfg.update('configopts', '--with-fcflags="%s"' % os.getenv('F90FLAGS'))
+
+            if not self.toolchain.comp_family() == toolchain.GCC:  # @UndefinedVariable
+                self.cfg.update('configopts', '--with-gnu-compilers=0')
+
+            # MPI
+            if self.toolchain.options.get('usempi', None):
+                self.cfg.update('configopts', '--with-mpi=1')
+
+            # build options
+            self.cfg.update('configopts', '--with-build-step-np=%s' % self.cfg['parallel'])
+            self.cfg.update('configopts', '--with-shared-libraries=%d' % self.cfg['shared_libs'])
+            self.cfg.update('configopts', '--with-debugging=%d' % self.toolchain.options['debug'])
+            self.cfg.update('configopts', '--with-pic=%d' % self.toolchain.options['pic'])
+            self.cfg.update('configopts', '--with-x=0 --with-windows-graphics=0')
+
+            # PAPI support
+            if self.cfg['with_papi']:
+                papi_inc = self.cfg['papi_inc']
+                papi_inc_file = os.path.join(papi_inc, "papi.h")
+                papi_lib = self.cfg['papi_lib']
+                if os.path.isfile(papi_inc_file) and os.path.isfile(papi_lib):
+                    self.cfg.update('configopts', '--with-papi=1')
+                    self.cfg.update('configopts', '--with-papi-include=%s' % papi_inc)
+                    self.cfg.update('configopts', '--with-papi-lib=%s' % papi_lib)
+                else:
+                    raise EasyBuildError("PAPI header (%s) and/or lib (%s) not found, can not enable PAPI support?",
+                                         papi_inc_file, papi_lib)
+
+            # Python extensions_step
+            if self.with_python:
+
+                # enable numpy support, but only if numpy is available
+                (_, ec) = run_cmd("python -c 'import numpy'", log_all=True, simple=False)
+                if ec == 0:
+                    self.cfg.update('configopts', '--with-numpy=1')
+
+                # enable mpi4py support, but only if mpi4py is available
+                (_, ec) = run_cmd("python -c 'import mpi4py'", log_all=True, simple=False)
+                if ec == 0:
+                    with_mpi4py_opt = '--with-mpi4py'
+                    if self.cfg['shared_libs'] and with_mpi4py_opt not in self.cfg['configopts']:
+                        self.cfg.update('configopts', '%s=1' % with_mpi4py_opt)
+
+            # FFTW, ScaLAPACK (and BLACS for older PETSc versions)
+            deps = ["ScaLAPACK"] #FFTW
+            if LooseVersion(self.version) < LooseVersion("3.5"):
+                deps.append("BLACS")
+            for dep in deps:
+                libdir = os.getenv('%s_LIB_DIR' % dep.upper())
+                libs = os.getenv('%s_STATIC_LIBS' % dep.upper())
+                if libdir and libs:
+                    with_arg = "--with-%s" % dep.lower()
+                    self.cfg.update('configopts', '%s=1' % with_arg)
+                    self.cfg.update('configopts', '%s-lib=[%s/%s]' % (with_arg, libdir, libs))
+
+                    inc = os.getenv('%s_INC_DIR' % dep.upper())
+                    if inc:
+                        self.cfg.update('configopts', '%s-include=%s' % (with_arg, inc))
+                else:
+                    self.log.info("Missing inc/lib info, so not enabling %s support." % dep)
+
+
+
+
+
+
+            # BLAS, LAPACK libraries
+            bl_libdir = os.getenv('BLAS_LAPACK_LIB_DIR')
+            bl_libs = os.getenv('BLAS_LAPACK_STATIC_LIBS')
+            if bl_libdir and bl_libs:
+                self.cfg.update('configopts', '--with-blas-lapack-lib=[%s/%s]' % (bl_libdir, bl_libs))
+            else:
+                raise EasyBuildError("One or more environment variables for BLAS/LAPACK not defined?")
+
+            # additional dependencies
+            # filter out deps handled seperately
+            sep_deps = ['BLACS', 'BLAS', 'CMake', 'LAPACK', 'numpy', #'FFTW'
+                        'mpi4py', 'papi', 'ScaLAPACK', 'SciPy-bundle', 'SuiteSparse']
+            # SCOTCH has to be treated separately since they add weird postfixes
+            # to library names from SCOTCH 7.0.1 or PETSc version 3.17.
+            if (LooseVersion(self.version) >= LooseVersion("3.17")):
+                sep_deps.append('SCOTCH')
+            depfilter = [d['name'] for d in self.cfg.builddependencies()] + sep_deps
+
+            deps = [dep['name'] for dep in self.cfg.dependencies() if not dep['name'] in depfilter]
+            for dep in deps:
+                if isinstance(dep, string_type):
+                    dep = (dep, dep)
+                deproot = get_software_root(dep[0])
+                if deproot:
+                    if (LooseVersion(self.version) >= LooseVersion("3.5")) and (dep[1] == "SCOTCH"):
+                        withdep = "--with-pt%s" % dep[1].lower()  # --with-ptscotch is the configopt PETSc >= 3.5
+                    else:
+                        withdep = "--with-%s" % dep[1].lower()
+                    self.cfg.update('configopts', '%s=1 %s-dir=%s' % (withdep, withdep, deproot))
+
+            # SCOTCH has to be treated separately since they add weird postfixes
+            # to library names from SCOTCH 7.0.1 or PETSc version 3.17.
+            scotch = get_software_root('SCOTCH')
+            scotch_ver = get_software_version('SCOTCH')
+            if (scotch and LooseVersion(scotch_ver) >= LooseVersion("7.0")):
+                withdep = "--with-ptscotch"
+                scotch_inc = [os.path.join(scotch, "include")]
+                inc_spec = "-include=[%s]" % ','.join(scotch_inc)
+
+                # For some reason there is a v3 suffix added to libptscotchparmetis
+                # which is the reason for this new code.
+                req_scotch_libs = ['libesmumps.a', 'libptesmumps.a', 'libptscotch.a',
+                                   'libptscotcherr.a', 'libptscotchparmetisv3.a', 'libscotch.a',
+                                   'libscotcherr.a']
+                scotch_libs = [os.path.join(scotch, "lib", x) for x in req_scotch_libs]
+                lib_spec = "-lib=[%s]" % ','.join(scotch_libs)
+                self.cfg.update('configopts', ' '.join([withdep + spec for spec in ['=1', inc_spec, lib_spec]]))
+
+            # SuiteSparse options changed in PETSc 3.5,
+            suitesparse = get_software_root('SuiteSparse')
+            if suitesparse:
+                if LooseVersion(self.version) >= LooseVersion("3.5"):
+                    withdep = "--with-suitesparse"
+                    # specified order of libs matters!
+                    ss_libs = ["UMFPACK", "KLU", "CHOLMOD", "BTF", "CCOLAMD", "COLAMD", "CAMD", "AMD"]
+                    # More libraries added after version 3.17
+                    if LooseVersion(self.version) >= LooseVersion("3.17"):
+                        # specified order of libs matters!
+                        ss_libs = ["UMFPACK", "KLU", "SPQR", "CHOLMOD", "BTF", "CCOLAMD",
+                                   "COLAMD", "CSparse", "CXSparse", "LDL", "RBio",
+                                   "SLIP_LU", "CAMD", "AMD"]
+
+                    suitesparse_inc = [os.path.join(suitesparse, x, "Include")
+                                       for x in ss_libs]
+                    suitesparse_inc.append(os.path.join(suitesparse, "SuiteSparse_config"))
+                    inc_spec = "-include=[%s]" % ','.join(suitesparse_inc)
+
+                    suitesparse_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.replace("_", "").lower())
+                                        for x in ss_libs]
+                    suitesparse_libs.append(os.path.join(suitesparse, "SuiteSparse_config", "libsuitesparseconfig.a"))
+                    lib_spec = "-lib=[%s]" % ','.join(suitesparse_libs)
+                else:
+                    # CHOLMOD and UMFPACK are part of SuiteSparse (PETSc < 3.5)
+                    withdep = "--with-umfpack"
+                    inc_spec = "-include=%s" % os.path.join(suitesparse, "UMFPACK", "Include")
+                    # specified order of libs matters!
+                    umfpack_libs = [os.path.join(suitesparse, x, "Lib", "lib%s.a" % x.lower())
+                                    for x in ["UMFPACK", "CHOLMOD", "COLAMD", "AMD"]]
+                    lib_spec = "-lib=[%s]" % ','.join(umfpack_libs)
+
+                self.cfg.update('configopts', ' '.join([withdep + spec for spec in ['=1', inc_spec, lib_spec]]))
+
+            # set PETSC_DIR for configure (env) and build_step
+            env.setvar('PETSC_DIR', self.cfg['start_dir'])
+            self.cfg.update('buildopts', 'PETSC_DIR=%s' % self.cfg['start_dir'])
+
+            if self.cfg['sourceinstall']:
+                # run configure without --prefix (required)
+                cmd = "%s ./configure %s" % (self.cfg['preconfigopts'], self.cfg['configopts'])
+                (out, _) = run_cmd(cmd, log_all=True, simple=False)
+            else:
+                out = super(EB_PETSc, self).configure_step()
+
+            # check for errors in configure
+            error_regexp = re.compile("ERROR")
+            if error_regexp.search(out):
+                raise EasyBuildError("Error(s) detected in configure output!")
+
+            if self.cfg['sourceinstall']:
+                # figure out PETSC_ARCH setting
+                petsc_arch_regex = re.compile(r"^\s*PETSC_ARCH:\s*(\S+)$", re.M)
+                res = petsc_arch_regex.search(out)
+                if res:
+                    self.petsc_arch = res.group(1)
+                    self.cfg.update('buildopts', 'PETSC_ARCH=%s' % self.petsc_arch)
+                else:
+                    raise EasyBuildError("Failed to determine PETSC_ARCH setting.")
+
+            self.petsc_subdir = '%s-%s' % (self.name.lower(), self.version)
+
+        else:  # old versions (< 3.x)
+
+            self.cfg.update('configopts', '--prefix=%s' % self.installdir)
+            self.cfg.update('configopts', '--with-shared=1')
+
+            # additional dependencies
+            for dep in ["SCOTCH"]:
+                deproot = get_software_root(dep)
+                if deproot:
+                    withdep = "--with-%s" % dep.lower()
+                    self.cfg.update('configopts', '%s=1 %s-dir=%s' % (withdep, withdep, deproot))
+
+            cmd = "./config/configure.py %s" % self.get_cfg('configopts')
+            run_cmd(cmd, log_all=True, simple=True)
+
+        # Make sure to set test_parallel before self.cfg['parallel'] is set to None
+        if self.cfg['test_parallel'] is None and self.cfg['parallel']:
+            self.cfg['test_parallel'] = self.cfg['parallel']
+
+        # PETSc > 3.5, make does not accept -j
+        # to control parallel build, we need to specify MAKE_NP=... as argument to 'make' command
+        if LooseVersion(self.version) >= LooseVersion("3.5"):
+            self.cfg.update('buildopts', "MAKE_NP=%s" % self.cfg['parallel'])
+            self.cfg['parallel'] = None
+
+    # default make should be fine
+
+    def test_step(self):
+        """
+        Test the compilation
+        """
+
+        # Each PETSc test may use multiple threads, so running "self.cfg['parallel']" of them may lead to
+        # some oversubscription every now and again. Not a big deal, but if needed a reduced parallelism
+        # can be specified with test_parallel - and it takes priority
+        paracmd = ''
+        self.log.info("In test_step: %s" % self.cfg['test_parallel'])
+        if self.cfg['test_parallel'] is not None:
+            paracmd = "-j %s" % self.cfg['test_parallel']
+
+        if self.cfg['runtest']:
+            cmd = "%s make %s %s %s" % (self.cfg['pretestopts'], paracmd, self.cfg['runtest'], self.cfg['testopts'])
+            (out, _) = run_cmd(cmd, log_all=True, simple=False)
+
+            return out
+
+    def install_step(self):
+        """
+        Install using make install (for non-source installations),
+        or by symlinking files (old versions, < 3).
+        """
+        if LooseVersion(self.version) >= LooseVersion("3"):
+            if not self.cfg['sourceinstall']:
+                super(EB_PETSc, self).install_step()
+                petsc_root = self.installdir
+            else:
+                petsc_root = os.path.join(self.installdir, self.petsc_subdir)
+            # Remove MPI-CXX flags added during configure to prevent them from being passed to consumers of PETsc
+            petsc_variables_path = os.path.join(petsc_root, 'lib', 'petsc', 'conf', 'petscvariables')
+            if os.path.isfile(petsc_variables_path):
+                fix = (r'^(CXX_FLAGS|CXX_LINKER_FLAGS|CONFIGURE_OPTIONS)( = .*)%s(.*)$' % NO_MPI_CXX_EXT_FLAGS,
+                       r'\1\2\3')
+                apply_regex_substitutions(petsc_variables_path, [fix])
+
+        else:  # old versions (< 3.x)
+
+            for fn in ['petscconf.h', 'petscconfiginfo.h', 'petscfix.h', 'petscmachineinfo.h']:
+                includedir = os.path.join(self.installdir, 'include')
+                bmakedir = os.path.join(self.installdir, 'bmake', 'linux-gnu-c-opt')
+                symlink(os.path.join(bmakedir, fn), os.path.join(includedir, fn))
+
+    def make_module_req_guess(self):
+        """Specify PETSc custom values for PATH, CPATH and LD_LIBRARY_PATH."""
+
+        guesses = super(EB_PETSc, self).make_module_req_guess()
+
+        guesses.update({
+            'CPATH': [os.path.join(self.prefix_lib, 'include'), os.path.join(self.prefix_inc, 'include')],
+            'LD_LIBRARY_PATH': [os.path.join(self.prefix_lib, 'lib')],
+            'PATH': [os.path.join(self.prefix_bin, 'bin')],
+            # see https://www.mcs.anl.gov/petsc/documentation/faq.html#sparse-matrix-ascii-format
+            'PYTHONPATH': [os.path.join('lib', 'petsc', 'bin')],
+        })
+
+        return guesses
+
+    def make_module_extra(self):
+        """Set PETSc specific environment variables (PETSC_DIR, PETSC_ARCH)."""
+        txt = super(EB_PETSc, self).make_module_extra()
+
+        if self.cfg['sourceinstall']:
+            txt += self.module_generator.set_environment('PETSC_DIR', os.path.join(self.installdir, self.petsc_subdir))
+            txt += self.module_generator.set_environment('PETSC_ARCH', self.petsc_arch)
+        else:
+            txt += self.module_generator.set_environment('PETSC_DIR', self.installdir)
+
+        return txt
+
+    def sanity_check_step(self):
+        """Custom sanity check for PETSc"""
+
+        if self.cfg['shared_libs']:
+            libext = get_shared_lib_ext()
+        else:
+            libext = 'a'
+
+        custom_paths = {
+            'files': [os.path.join(self.prefix_lib, 'lib', 'libpetsc.%s' % libext)],
+            'dirs': [os.path.join(self.prefix_bin, 'bin'), os.path.join(self.prefix_inc, 'include'),
+                     os.path.join(self.prefix_lib, 'include')]
+        }
+        if LooseVersion(self.version) < LooseVersion('3.6'):
+            custom_paths['dirs'].append(os.path.join(self.prefix_lib, 'conf'))
+        else:
+            custom_paths['dirs'].append(os.path.join(self.prefix_lib, 'lib', 'petsc', 'conf'))
+
+        custom_commands = []
+        if self.with_python:
+            custom_commands.append("python -m PetscBinaryIO --help")
+
+        super(EB_PETSc, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
+

Golden_Repo/p/PETSc/PETSc-3.18.5-gpsmkl-2022a.eb

+name = "PETSc"
+version = "3.18.5"
+
+homepage = 'http://www.mcs.anl.gov/petsc'
+description = """PETSc, pronounced PET-see (the S is silent), is a suite
+of data structures and routines for the scalable (parallel) solution
+of scientific applications modeled by partial differential equations.
+
+This version is configured with several downloads of other libraries,
+with --with-large-file-io and no debugging. It is a C and Fortran
+version with default 4-Byte integer values.
+
+For more information see $PETSC_DIR/lib/petsc/conf/configure-hash.
+"""
+
+toolchain = {'name': 'gpsmkl', 'version': '2022a'}
+
+toolchainopts = {'usempi': True, 'pic': True}
+
+source_urls = ['http://ftp.mcs.anl.gov/pub/petsc/release-snapshots']
+sources = ['petsc-%s.tar.gz' % version]
+checksums = ['df73ae13a4c5758325a9d69350cac423742657d8a8fc5782504b0e469ce46499']
+
+builddependencies = [
+    ('CMake', '3.23.1')
+]
+
+download_deps = [
+    'triangle',
+]
+
+dependencies = [
+    ('HDF5', '1.12.2'),
+    ('METIS', '5.1.0'),
+    ('ParMETIS', '4.0.3')
+]
+
+download_deps_static = [
+    'hypre',
+    'spooles',
+    'superlu',
+    'superlu_dist',
+    'mumps',
+    'spai',
+    'chaco',
+    'sundials2',
+    'parms',
+]
+
+configopts = '--with-large-file-io --with-cxx-dialect=C++11'
+configopts += '--with-fftw=1 --with-fftw-lib=[/p/software/jurecadc/stages/2023/software/\
+imkl-FFTW/2022.1.0-gpsmpi-2022a/lib/libfftw3xc_gnu_pic.a,\
+libfftw3x_cdft_lp64_pic.a,libmkl_cdft_core.a,libmkl_blacs_intelmpi_lp64.a,\
+libmkl_gf_lp64.a,libmkl_sequential.a,libmkl_core.a,libgfortran.a]'
+configopts += '--with-fftw-include=[/p/software/jurecadc/stages/2023/software/imkl/2022.1.0/mkl/2022.1.0/include/fftw]'
+shared_libs = 1
+
+moduleclass = 'numlib'