update

0_hello_world/CMakeLists.txt

+add_executable(hello_world.exe hello_world.cpp)
+target_link_libraries(hello_world.exe PRIVATE mkl_common_mt -fopenmp) 
+

1_integral/CMakeLists.txt

+add_executable(integral.exe integral.cpp)
+target_link_libraries(integral.exe PRIVATE ${OPENMP_FLAGS}) 
+

2_gemm/CMakeLists.txt

+add_executable(gemm_mkl.exe gemm_mkl.cpp)
+target_link_libraries(gemm_mkl.exe PRIVATE mkl_common_mt) 
+

2_gemm/gemm_mkl.cpp

+#include <stdio.h>
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include <time.h>
+#include <iomanip>
+#include <chrono>
+#include <cstdlib>
+#include <boost/program_options.hpp>
+#include <boost/foreach.hpp>
+#include <boost/tokenizer.hpp>
+
+#include <mkl.h>
+
+#include <omp.h>
+
+int main(int argc, char** argv) {
+
+    using namespace boost::program_options;
+
+    options_description desc_cmdline;
+
+    desc_cmdline.add_options()
+        ("size,s", value<int>()->default_value(-1), "Matrix size (if > 0, overrides m, n, k).")
+        ("m", value<int>()->default_value(1024), "Size m.")
+        ("n", value<int>()->default_value(1024), "Size n.")
+        ("k", value<int>()->default_value(512), "Size k.")
+        ("omp:threads", value<int>()->default_value(1), "OpenMP thread number.")
+        ("repeat", value<int>()->default_value(1), "repeat times.");
+
+    variables_map vm;
+
+    store(parse_command_line(argc, argv, desc_cmdline),vm);
+
+    int m = vm["m"].as<int>();
+    int n = vm["n"].as<int>();
+    int k = vm["k"].as<int>();
+    int s = vm["size"].as<int>();
+    int repeat = vm["repeat"].as<int>();
+
+    if (s > 0)
+        m = n = k = s;
+
+    int num_threads = vm["omp:threads"].as<int>();
+
+    const double alpha = 1.0;
+
+    const double beta = 1.0;
+
+    std::vector<double> A(m * k);
+    std::vector<double> B(k * n);
+    std::vector<double> C(m * n);
+
+    std::generate(A.begin(), A.end(), [ttt = 1] () mutable { return ttt++; });
+    std::generate(B.begin(), B.end(), [ttt = 2] () mutable { return ttt++; });
+    std::fill(C.begin(), C.end(), (double)0.0);
+
+    double flops = 2.0 * m * n * k / 1e9;
+
+    std::chrono::high_resolution_clock::time_point start, end;
+
+    std::chrono::duration<double> elapsed;
+
+    start = std::chrono::high_resolution_clock::now();
+
+    for(int i = 0; i < repeat; i++){
+
+        dgemm("N", "N", &m, &n, &k, &alpha, A.data(), &m, B.data(), &k, &beta, C.data(), &m);
+
+    }
+
+    end = std::chrono::high_resolution_clock::now();
+
+    elapsed = std::chrono::duration_cast<std::chrono::duration<double>>(end - start);
+
+    std::cout << m << "," << n << "," << k << "," << "GEMM (mkl)," << num_threads << "," << 1 << "," << elapsed.count() << "," <<  repeat * flops / elapsed.count() << std::endl;
+
+    return 0;
+}
+

CMakeLists.txt

+cmake_minimum_required(VERSION 3.0)
+
+project(openmp CXX)
+
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE Release)
+endif()
+
+set(CMAKE_CXX_FLAGS_DEBUG "-g")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3")
+
+list (APPEND CMAKE_MODULE_PATH "${openmp_SOURCE_DIR}/cmake-modules")
+
+option(MKL_MT "MKL with multithreading support" ON)
+
+if(MKL_MT)
+  if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    set(OPENMP_FLAGS "-fopenmp")
+  elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set(OPENMP_FLAGS "-fopenmp")
+  elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+    set(OPENMP_FLAGS "-openmp")
+  elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+    set(OPENMP_FLAGS "-fopenmp")
+  endif()
+endif()
+
+find_package(MKL REQUIRED)
+
+if (MKL_FOUND)
+    set(LAPACK_INCLUDE_DIRS ${MKL_INCLUDE_DIRS})
+    if(MKL_MT)
+      set(LAPACK_LIBRARIES_MT
+        mkl_intel_lp64 mkl_core mkl_gnu_thread
+        gomp pthread m dl mkl_blacs_intelmpi_lp64
+        mkl_scalapack_lp64
+        CACHE PATH "LAPACK_LIBS"
+      )
+    else()   
+      set(LAPACK_LIBRARIES
+        mkl_intel_lp64 mkl_core mkl_sequential
+        pthread m dl mkl_blacs_intelmpi_lp64
+        mkl_scalapack_lp64
+        CACHE PATH "LAPACK_LIBS"
+      )
+    endif()
+    link_directories(${MKL_LIB_DIR})
+endif()
+
+find_package(Boost REQUIRED COMPONENTS program_options)
+
+include(CheckCXXSymbolExists)
+CHECK_CXX_SYMBOL_EXISTS(MPI_Finalize mpi.h COMPILER_HAS_MPI)
+
+if(COMPILER_HAS_MPI)
+  add_library(MPI::MPI_C INTERFACE IMPORTED)
+else()
+  find_package(MPI REQUIRED)
+  if(MPI_FOUND)
+    SET(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
+  endif()
+endif()
+
+add_library(mkl_common INTERFACE)
+target_include_directories(mkl_common
+  INTERFACE
+    ${LAPACK_INCLUDE_DIRS})
+
+target_link_libraries(mkl_common
+  INTERFACE
+    ${LAPACK_LIBRARIES}
+    Boost::program_options)
+
+if(MKL_MT)
+add_library(mkl_common_mt INTERFACE)
+  target_include_directories(mkl_common_mt
+    INTERFACE
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+      $<INSTALL_INTERFACE:include>
+      ${LAPACK_INCLUDE_DIRS})
+
+  target_link_libraries(mkl_common_mt
+    INTERFACE
+      ${LAPACK_LIBRARIES_MT}
+      Boost::program_options)
+endif()
+
+add_library(mkl_common_with_mpi INTERFACE)
+
+if(COMPILER_HAS_MPI)
+  target_link_libraries(mkl_common_with_mpi
+    INTERFACE
+    mkl_common
+    MPI::MPI_C)
+else()
+  target_link_libraries(mkl_common_with_mpi
+    INTERFACE
+    mkl_common
+    ${MPI_CXX_LIBRARIES})
+endif()
+
+SET(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
+
+ADD_SUBDIRECTORY(0_hello_world)
+ADD_SUBDIRECTORY(1_integral)
+
+include(Dart)
+include(CPack)
+
+

README.md

@@ -2,6 +2,10 @@
 
 OpenMP test codes
 
+## To run with hpxMP on JUWELS
+
+1. load the modules
+
 ```bash
 module load GCC
 module load ParaStationMPI/5.2.2-1
@@ -10,4 +14,16 @@ module load CMake/3.14.0
 module load Boost/1.69.0-Python-2.7.16
 module load CUDA/10.1.105
 module load hwloc/2.1.0
+module load imkl
+```
+2. compile the code with normal openMP flags by GCC or others
+
+```bash
+gcc -fopenmp my_application.c -o my_application
+```
+
+3. run the application with corresponding hpxMP library
+
+```bash
+LD_PRELOAD=/p/project/cslai/wu/source/hpxMP/hpxMP-0.3.0/build/gcc_juwels/libhpxmp.so ./my_application
 ```

cmake-modules/FindBLIS.cmake

+# - Find BLIS
+# Find the BLIS libraries
+# BLIS: BLAS-like Library Instantiation Software Framework
+# Github: https://github.com/flame/blis
+#   BLIS_FOUND            : True if BLIS_INCUDE_DIR are found
+#   BLIS_INCLUDE_DIR      : where to find blis.h, etc.
+#   BLIS_INCLUDE_DIRS     : set when BLIS_INCLUDE_DIR found
+#   BLIS_LIBRARIES        : the library to link against.
+
+set(BLIS_ROOT "/usr/local" CACHE PATH "Folder containing BLIS libraries")
+
+if (NOT BLIS_ROOT AND DEFINED ENV{BLIS_ROOT})
+  set(BLIS_ROOT $ENV{BLIS_ROOT} CACHE PATH "Folder containing BLIS")
+elseif (NOT BLIS_ROOT AND DEFINED ENV{BLISROOT})
+  set(BLIS_ROOT $ENV{BISLROOT} CACHE PATH "Folder containing BLIS")
+endif()
+
+find_path(BLIS_INCLUDE_DIR
+    NAMES
+	blis.h
+    PATHS
+	${BLIS_ROOT}/include/blis
+)
+
+set(_BLIS_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+
+if(WIN32)
+    if(BLIS_STATIC)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES .lib)
+    else()
+        set(CMAKE_FIND_LIBRARY_SUFFIXES _dll.lib)
+    endif()
+else()
+    if(BLIS_STATIC)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
+    else()
+        if(APPLE)
+            set(CMAKE_FIND_LIBRARY_SUFFIXES .dylib)
+        else()
+            set(CMAKE_FIND_LIBRARY_SUFFIXES .so)
+        endif()
+    endif()
+endif()
+
+find_library(BLIS_LIBRARY blis
+   PATHS ${BLIS_ROOT}/lib
+)
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(BLIS
+	DEFAULT_MSG
+	BLIS_LIBRARY
+	BLIS_INCLUDE_DIR
+)
+
+if(BLIS_FOUND)
+    set(BLIS_INCLUDE_DIRS ${BLIS_INCLUDE_DIR})
+    set(BLIS_LIBRARIES ${BLIS_LIBRARY}) 
+endif()
+
+
+
+

cmake-modules/FindElemental.cmake

+# - Find Elemental 
+# Find the Elemental libraries
+# Elemental: Distributed-memory, arbitrary-precision, dense and sparse-direct 
+# linear algebra, conic optimization, and lattice reduction 
+# Github: https://github.com/elemental/Elemental 
+#   Elemental_FOUND            : True if Elemental_INCUDE_DIR are found
+#   Elemental_INCLUDE_DIR      : where to find elemental.h, etc.
+#   Elemental_INCLUDE_DIRS     : set when Elemental_INCLUDE_DIR found
+#   Elemental_LIBRARIES        : the library to link against.
+#   PMRRR_LIBRARY	       : PMRRR_LIBRARY to link
+
+set(Elemental_ROOT "/usr/local" CACHE PATH "Folder containing Elemental libraries")
+
+if (NOT Elemental_ROOT AND DEFINED ENV{Elemental_ROOT})
+  set(Elemental_ROOT $ENV{Elemental_ROOT} CACHE PATH "Folder containing Elemental")
+elseif (NOT Elemental_ROOT AND DEFINED ENV{ElementalROOT})
+  set(Elemental_ROOT $ENV{ElementalROOT} CACHE PATH "Folder containing Elemental")
+endif()
+
+find_file(Elemental_basic_header 
+    NAMES
+        elemental.hpp
+    PATHS
+	${Elemental_ROOT}/include
+)
+
+if(Elemental_basic_header)
+
+  find_path(Elemental_INCLUDE_DIR
+    NAMES
+        elemental.hpp
+    PATHS
+        ${Elemental_ROOT}/include
+  )
+
+  find_library(Elemental_LIBRARY libelemental.a
+      PATHS ${Elemental_ROOT}/lib
+  )
+
+  find_library(PMRRR_LIBRARY libpmrrr.a
+     PATHS ${Elemental_ROOT}/lib
+  )
+  
+  set(LOW_VERISON true)
+
+else()
+
+  find_path(Elemental_INCLUDE_DIR
+    NAMES
+	El.hpp
+    PATHS
+        ${Elemental_ROOT}/include
+  )
+
+  find_library(Elemental_LIBRARY libEl.so 
+      PATHS ${Elemental_ROOT}/lib
+  )
+
+  find_library(PMRRR_LIBRARY libpmrrr.so
+     PATHS ${Elemental_ROOT}/lib
+  )
+
+  set(LOW_VERISON false)
+
+endif()
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(Elemental
+	DEFAULT_MSG
+	Elemental_LIBRARY
+	Elemental_INCLUDE_DIR
+)
+
+if(Elemental_FOUND)
+    set(Elemental_INCLUDE_DIRS ${Elemental_INCLUDE_DIR})
+    set(Elemental_LIBRARIES ${Elemental_LIBRARY}) 
+    set(Elemental_IF_LOW_VERSION ${LOW_VERISON})
+endif()
+
+
+

cmake-modules/FindHWLOC.cmake

+# FindHwloc
+# ----------
+#
+#
+# You may declare HWLOC_ROOT environment variable to tell where
+# your hwloc library is installed. 
+#
+# Once done this will define::
+#
+#   Hwloc_FOUND            - True if hwloc was found
+#   Hwloc_INCLUDE_DIRS     - include directories for hwloc
+#   Hwloc_LIBRARIES        - link against these libraries to use hwloc
+#   Hwloc_VERSION          - version
+#   Hwloc_CFLAGS           - include directories as compiler flags
+#   Hwloc_LDLFAGS          - link paths and libs as compiler flags
+#
+
+if(WIN32)
+  find_path(Hwloc_INCLUDE_DIR
+    NAMES
+      hwloc.h
+    PATHS
+      ENV "PROGRAMFILES(X86)"
+      ENV HWLOC_ROOT
+    PATH_SUFFIXES
+      include
+  )
+
+  find_library(Hwloc_LIBRARY
+    NAMES 
+      libhwloc.lib
+    PATHS
+      ENV "PROGRAMFILES(X86)"
+      ENV HWLOC_ROOT
+    PATH_SUFFIXES
+      lib
+  )
+
+  #
+  # Check if the found library can be used to linking 
+  #
+  SET (_TEST_SOURCE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/linktest.c")
+  FILE (WRITE "${_TEST_SOURCE}"
+    "
+    #include <hwloc.h>
+    int main()
+    { 
+      hwloc_topology_t topology;
+      int nbcores;
+      hwloc_topology_init(&topology);
+      hwloc_topology_load(topology);
+      nbcores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
+      hwloc_topology_destroy(topology);
+      return 0;
+    }
+    "
+  )
+
+  TRY_COMPILE(_LINK_SUCCESS ${CMAKE_BINARY_DIR} "${_TEST_SOURCE}"
+    CMAKE_FLAGS
+    "-DINCLUDE_DIRECTORIES:STRING=${Hwloc_INCLUDE_DIR}"
+    CMAKE_FLAGS
+    "-DLINK_LIBRARIES:STRING=${Hwloc_LIBRARY}"
+  )
+
+  IF(NOT _LINK_SUCCESS)
+    if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+      message(STATUS "You are building 64bit target.")
+    ELSE()
+      message(STATUS "You are building 32bit code. If you like to build x64 use e.g. -G 'Visual Studio 12 Win64' generator." )
+    ENDIF()
+    message(FATAL_ERROR "Library found, but linking test program failed.")
+  ENDIF()
+
+  #
+  # Resolve version if some compiled binary found...
+  #
+  find_program(HWLOC_INFO_EXECUTABLE
+    NAMES 
+      hwloc-info
+    PATHS
+      ENV HWLOC_ROOT 
+    PATH_SUFFIXES
+      bin
+  )
+  
+  if(HWLOC_INFO_EXECUTABLE)
+    execute_process(
+      COMMAND ${HWLOC_INFO_EXECUTABLE} "--version" 
+      OUTPUT_VARIABLE HWLOC_VERSION_LINE 
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    string(REGEX MATCH "([0-9]+.[0-9]+)$" 
+      Hwloc_VERSION "${HWLOC_VERSION_LINE}")
+    unset(HWLOC_VERSION_LINE)
+  endif()
+  
+  #
+  # All good
+  #
+
+  set(Hwloc_LIBRARIES ${Hwloc_LIBRARY})
+  set(Hwloc_INCLUDE_DIRS ${Hwloc_INCLUDE_DIR})
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(
+    Hwloc
+    FOUND_VAR Hwloc_FOUND
+    REQUIRED_VARS Hwloc_LIBRARY Hwloc_INCLUDE_DIR
+    VERSION_VAR Hwloc_VERSION)
+
+  mark_as_advanced(
+    Hwloc_INCLUDE_DIR
+    Hwloc_LIBRARY)
+
+  foreach(arg ${Hwloc_INCLUDE_DIRS})
+    set(Hwloc_CFLAGS "${Hwloc_CFLAGS} /I${arg}")
+  endforeach()
+
+  set(Hwloc_LDFLAGS "${Hwloc_LIBRARY}")
+
+else()
+
+  if(CMAKE_CROSSCOMPILING)
+
+  find_path(Hwloc_INCLUDE_DIRS
+    NAMES
+      hwloc.h
+    PATHS
+      ENV HWLOC_ROOT
+  )
+
+  find_library(Hwloc_LIBRARIES
+    NAMES
+      hwloc
+    PATHS
+      ENV HWLOC_ROOT
+  )
+
+  if(Hwloc_INCLUDE_DIRS AND Hwloc_LIBRARIES)
+    message(WARNING "HWLOC library found using find_library() - cannot determine version. Assuming 1.7.0")
+    set(Hwloc_FOUND 1)
+    set(Hwloc_VERSION "1.7.0")
+  endif()
+
+  else() # Find with pkgconfig for non-crosscompile builds
+
+  find_package(PkgConfig)
+
+  if(HWLOC_ROOT)
+    set(ENV{PKG_CONFIG_PATH} "${HWLOC_ROOT}/lib/pkgconfig")
+  else()
+    foreach(PREFIX ${CMAKE_PREFIX_PATH})
+      set(PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${PREFIX}/lib/pkgconfig")
+    endforeach()
+    set(ENV{PKG_CONFIG_PATH} "${PKG_CONFIG_PATH}:$ENV{PKG_CONFIG_PATH}")
+  endif()
+
+  if(hwloc_FIND_REQUIRED)
+    set(_hwloc_OPTS "REQUIRED")
+  elseif(hwloc_FIND_QUIETLY)
+    set(_hwloc_OPTS "QUIET")
+  else()
+    set(_hwloc_output 1)
+  endif()
+
+  if(hwloc_FIND_VERSION)
+    if(hwloc_FIND_VERSION_EXACT)
+      pkg_check_modules(Hwloc ${_hwloc_OPTS} hwloc=${hwloc_FIND_VERSION})
+    else()
+      pkg_check_modules(Hwloc ${_hwloc_OPTS} hwloc>=${hwloc_FIND_VERSION})
+    endif()
+  else()
+    pkg_check_modules(Hwloc ${_hwloc_OPTS} hwloc)
+  endif()
+
+  if(Hwloc_FOUND)
+    include(FindPackageHandleStandardArgs)
+    find_package_handle_standard_args(Hwloc DEFAULT_MSG Hwloc_LIBRARIES)
+
+    if(NOT ${Hwloc_VERSION} VERSION_LESS 1.7.0)
+      set(Hwloc_GL_FOUND 1)
+    endif()
+
+    if(_hwloc_output)
+      message(STATUS
+        "Found hwloc ${Hwloc_VERSION} in ${Hwloc_INCLUDE_DIRS}:${Hwloc_LIBRARIES}")
+    endif()
+  endif()
+
+  endif() # cross-compile else
+
+endif()
+
+

cmake-modules/FindMKL.cmake

+# - Find Intel MKL
+# Find the MKL libraries
+#
+# Options:
+#
+#   MKL_ROOT              : place to search for MKL
+#   $ENV{MKL_ROOT}        :   "  "  "  "  "  "  "
+#   $ENV{MKLROOT}         :   "  "  "  "  "  "  "
+#   MKL_STATIC            : use static linking
+#   MKL_MULTI_THREADED    : use multi-threading
+#   MKL_SDL               : Single Dynamic Library interface
+#   MKL_32                : search for ia32 instead of intel64
+#   MKL_VERBOSE           : display info about found heders/libs/etc
+#
+# This module defines the following variables:
+#
+#   MKL_FOUND            : True if MKL_INCLUDE_DIR are found
+#   MKL_INCLUDE_DIR      : where to find mkl.h, etc.
+#   MKL_INCLUDE_DIRS     : set when MKL_INCLUDE_DIR found
+#   MKL_LIB_DIR          : the dir where the mkl libs are found
+#   MKL_LIBRARIES        : the library to link against.
+#   MKL_LIBRARIES_MT     : the library to link against.
+
+macro(MKL_MESSAGE string)
+  if (MKL_VERBOSE)
+    message("MKL: " ${string})
+  endif()
+endmacro()
+
+include(FindPackageHandleStandardArgs)
+
+set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder containing intel libs")
+if (NOT MKL_ROOT AND DEFINED ENV{MKL_ROOT})
+  set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder containing MKL")
+elseif (NOT MKL_ROOT AND DEFINED ENV{MKLROOT})
+  set(MKL_ROOT $ENV{MKLROOT} CACHE PATH "Folder containing MKL")
+elseif (NOT MKL_ROOT)
+  set(MKL_ROOT ${INTEL_ROOT}/mkl CACHE PATH "Folder containing MKL")
+endif()
+
+MKL_MESSAGE("Using MKL_ROOT ${MKL_ROOT}")
+
+# Find include dir
+find_path(MKL_INCLUDE_DIR mkl.h
+  PATHS ${MKL_ROOT}/include)
+
+MKL_MESSAGE("MKL_INCLUDE_DIR is ${MKL_INCLUDE_DIR}")
+
+# Find include directory
+#  There is no include folder under linux
+if(WIN32)
+    find_path(INTEL_INCLUDE_DIR omp.h
+        PATHS ${INTEL_ROOT}/include)
+    set(MKL_INCLUDE_DIR ${MKL_INCLUDE_DIR} ${INTEL_INCLUDE_DIR})
+endif()
+
+# Find libraries
+
+# Handle suffix
+set(_MKL_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+
+if(WIN32)
+    if(MKL_STATIC)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES .lib)
+    else()
+        set(CMAKE_FIND_LIBRARY_SUFFIXES _dll.lib)
+    endif()
+else()
+    if(MKL_STATIC)
+        set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
+    else()
+        if(APPLE)
+            set(CMAKE_FIND_LIBRARY_SUFFIXES .dylib)
+        else()
+            set(CMAKE_FIND_LIBRARY_SUFFIXES .so)
+        endif()
+    endif()
+endif()
+
+if(UNIX AND NOT APPLE)
+    if(MKL_32)
+        set(MKL_SUBDIR ia32)
+    else()
+        set(MKL_SUBDIR intel64)
+        MKL_MESSAGE("MKL sub dir is ${MKL_SUBDIR}")
+    endif()
+endif()
+
+set(MKL_LIB_DIR "${MKL_ROOT}/lib/${MKL_SUBDIR}" CACHE INTERNAL "lib path")
+MKL_MESSAGE("MKL_LIB_DIR sub dir is ${MKL_LIB_DIR}")
+
+# MKL is composed by four layers: Interface, Threading, Computational and RTL
+
+if(MKL_SDL)
+    find_library(MKL_LIBRARY mkl_rt
+        PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+
+    set(MKL_MINIMAL_LIBRARY ${MKL_LIBRARY})
+else()
+    ######################### Interface layer #######################
+    if(WIN32)
+        set(MKL_INTERFACE_LIBNAME mkl_intel_c)
+    else()
+        set(MKL_INTERFACE_LIBNAME mkl_intel_ilp64)
+    endif()
+
+    find_library(MKL_INTERFACE_LIBRARY ${MKL_INTERFACE_LIBNAME}
+        PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+    MKL_MESSAGE("MKL_INTERFACE_LIBRARY is ${MKL_INTERFACE_LIBRARY}")
+
+    ######################## Threading layer ########################
+    find_library(MKL_THREADING_LIBRARY mkl_sequential
+        PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+    find_library(MKL_THREADING_LIBRARY_MT mkl_gnu_thread
+        PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+
+    ####################### Computational layer #####################
+    find_library(MKL_CORE_LIBRARY mkl_core
+        PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+    find_library(MKL_FFT_LIBRARY mkl_cdft_core
+        PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+    find_library(MKL_SCALAPACK_LIBRARY mkl_scalapack_ilp64
+        PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+    find_library(MKL_BLACS_LIBRARY mkl_blacs_intelmpi_ilp64
+        PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+
+
+    MKL_MESSAGE("MKL_CORE_LIBRARY is ${MKL_CORE_LIBRARY}")
+    MKL_MESSAGE("MKL_FFT_LIBRARY is ${MKL_FFT_LIBRARY}")
+    MKL_MESSAGE("MKL_SCALAPACK_LIBRARY is ${MKL_SCALAPACK_LIBRARY}")
+    MKL_MESSAGE("MKL_BLACS_LIBRARY is ${MKL_BLACS_LIBRARY}")
+
+    ############################ RTL layer ##########################
+    if(WIN32)
+        set(MKL_RTL_LIBNAME libiomp5md)
+    else()
+        set(MKL_RTL_LIBNAME libiomp5)
+    endif()
+    find_library(MKL_RTL_LIBRARY ${MKL_RTL_LIBNAME}
+        PATHS ${INTEL_RTL_ROOT}/lib)
+    MKL_MESSAGE("MKL_RTL_LIBRARY is ${MKL_RTL_LIBRARY}")
+
+    set(MKL_LIBRARY
+        ${MKL_INTERFACE_LIBRARY}
+        ${MKL_CORE_LIBRARY}
+        ${MKL_THREADING_LIBRARY}
+        ${MKL_FFT_LIBRARY}
+        ${MKL_SCALAPACK_LIBRARY}
+    )
+    set(MKL_MINIMAL_LIBRARY ${MKL_INTERFACE_LIBRARY} ${MKL_THREADING_LIBRARY} ${MKL_CORE_LIBRARY})
+    if (MKL_RTL_LIBRARY)
+      set(MKL_LIBRARY ${MKL_LIBRARY} ${MKL_RTL_LIBRARY})
+      set(MKL_MINIMAL_LIBRARY ${MKL_MINIMAL_LIBRARY} ${MKL_RTL_LIBRARY})
+    endif()
+endif()
+
+set(CMAKE_FIND_LIBRARY_SUFFIXES ${_MKL_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
+
+find_package_handle_standard_args(MKL DEFAULT_MSG
+    MKL_INCLUDE_DIR MKL_LIBRARY MKL_MINIMAL_LIBRARY)
+
+if(MKL_FOUND)
+    set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR})
+    set(MKL_LIBRARIES ${MKL_LIBRARY})
+    set(MKL_MINIMAL_LIBRARIES ${MKL_LIBRARY})
+    set(MKL_LIBRARIES_MT
+        ${MKL_INTERFACE_LIBRARY}
+        ${MKL_CORE_LIBRARY}
+        ${MKL_THREADING_LIBRARY_MT}
+        ${MKL_FFT_LIBRARY}
+        ${MKL_SCALAPACK_LIBRARY}
+        ${MKL_BLACS_LIBRARY}
+    )
+    MKL_MESSAGE("MKL_LIBRARIES_MT are ${MKL_LIBRARIES_MT}")
+endif()
+
+

cmake-modules/FindPAPI.cmake

+# - Find PAPI
+# Find the PAPI libraries
+# PAPI: Performance Application Programming Interface
+# Website: https://icl.utk.edu/papi/index.html
+#   PAPI_FOUND            : True if PAPI_INCUDE_DIR are found
+#   PAPI_INCLUDE_DIR      : where to find papi.h, etc.
+#   PAPI_INCLUDE_DIRS     : set when PAPU_INCLUDE_DIR found
+#   PAPI_LIBRARIES        : the library to link against.
+
+set(PAPI_ROOT "/usr/local" CACHE PATH "Folder containing PAPI libraries")
+
+if (NOT PAPI_ROOT AND DEFINED ENV{PAPI_ROOT})
+  set(PAPI_ROOT $ENV{PAPI_ROOT} CACHE PATH "Folder containing PAPI")
+elseif (NOT PAPI_ROOT AND DEFINED ENV{PAPIROOT})
+  set(PAPI_ROOT $ENV{PAPIROOT} CACHE PATH "Folder containing PAPI")
+endif()
+
+find_path(PAPI_INCLUDE_DIR
+    NAMES
+        papi.h
+    PATHS
+        ${PAPI_ROOT}/include
+)
+
+find_library(PAPI_LIBRARY libpapi.so libpapi.a papi
+   PATHS ${PAPI_ROOT}/lib
+)
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(PAPI
+        DEFAULT_MSG
+        PAPI_LIBRARY
+        PAPI_INCLUDE_DIR
+)
+
+if(PAPI_FOUND)
+    set(PAPI_INCLUDE_DIRS ${PAPI_INCLUDE_DIR})
+    set(PAPI_LIBRARIES ${PAPI_LIBRARY})
+endif()
+
+