diff --git a/0_hello_world/CMakeLists.txt b/0_hello_world/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..512c3cbb07d4f2570d78b4d5299a0e96133a37aa
--- /dev/null
+++ b/0_hello_world/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_executable(hello_world.exe hello_world.cpp)
+target_link_libraries(hello_world.exe PRIVATE mkl_common_mt -fopenmp)
+
diff --git a/1_integral/CMakeLists.txt b/1_integral/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..787bf71f92aa8399e7e0caeca688d0aff7f68087
--- /dev/null
+++ b/1_integral/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_executable(integral.exe integral.cpp)
+target_link_libraries(integral.exe PRIVATE ${OPENMP_FLAGS})
+
diff --git a/2_gemm/CMakeLists.txt b/2_gemm/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ae7b71ff1c286e5c5fb761d0c5f15bdc25e8d0f9
--- /dev/null
+++ b/2_gemm/CMakeLists.txt
@@ -0,0 +1,3 @@
+add_executable(gemm_mkl.exe gemm_mkl.cpp)
+target_link_libraries(gemm_mkl.exe PRIVATE mkl_common_mt)
+
diff --git a/2_gemm/gemm_mkl.cpp b/2_gemm/gemm_mkl.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..053e4969303ed456749c41b62fcdc62d05470596
--- /dev/null
+++ b/2_gemm/gemm_mkl.cpp
@@ -0,0 +1,80 @@
+#include <stdio.h>
+#include <iostream>
+#include <vector>
+#include <cmath>
+#include <time.h>
+#include <iomanip>
+#include <chrono>
+#include <cstdlib>
+#include <boost/program_options.hpp>
+#include <boost/foreach.hpp>
+#include <boost/tokenizer.hpp>
+
+#include <mkl.h>
+
+#include <omp.h>
+
+int main(int argc, char** argv) {
+
+ using namespace boost::program_options;
+
+ options_description desc_cmdline;
+
+ desc_cmdline.add_options()
+ ("size,s", value<int>()->default_value(-1), "Matrix size (if > 0, overrides m, n, k).")
+ ("m", value<int>()->default_value(1024), "Size m.")
+ ("n", value<int>()->default_value(1024), "Size n.")
+ ("k", value<int>()->default_value(512), "Size k.")
+ ("omp:threads", value<int>()->default_value(1), "OpenMP thread number.")
+ ("repeat", value<int>()->default_value(1), "repeat times.");
+
+ variables_map vm;
+
+ store(parse_command_line(argc, argv, desc_cmdline),vm);
+
+ int m = vm["m"].as<int>();
+ int n = vm["n"].as<int>();
+ int k = vm["k"].as<int>();
+ int s = vm["size"].as<int>();
+ int repeat = vm["repeat"].as<int>();
+
+ if (s > 0)
+ m = n = k = s;
+
+ int num_threads = vm["omp:threads"].as<int>();
+
+ const double alpha = 1.0;
+
+ const double beta = 1.0;
+
+ std::vector<double> A(m * k);
+ std::vector<double> B(k * n);
+ std::vector<double> C(m * n);
+
+ std::generate(A.begin(), A.end(), [ttt = 1] () mutable { return ttt++; });
+ std::generate(B.begin(), B.end(), [ttt = 2] () mutable { return ttt++; });
+ std::fill(C.begin(), C.end(), (double)0.0);
+
+ double flops = 2.0 * m * n * k / 1e9;
+
+ std::chrono::high_resolution_clock::time_point start, end;
+
+ std::chrono::duration<double> elapsed;
+
+ start = std::chrono::high_resolution_clock::now();
+
+ for(int i = 0; i < repeat; i++){
+
+ dgemm("N", "N", &m, &n, &k, &alpha, A.data(), &m, B.data(), &k, &beta, C.data(), &m);
+
+ }
+
+ end = std::chrono::high_resolution_clock::now();
+
+ elapsed = std::chrono::duration_cast<std::chrono::duration<double>>(end - start);
+
+ std::cout << m << "," << n << "," << k << "," << "GEMM (mkl)," << num_threads << "," << 1 << "," << elapsed.count() << "," << repeat * flops / elapsed.count() << std::endl;
+
+ return 0;
+}
+
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..80cd1c0488b45552d464d57e19d7a51f52a3b0df
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,114 @@
+cmake_minimum_required(VERSION 3.0)
+
+project(openmp CXX)
+
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX_EXTENSIONS OFF)
+
+if(NOT CMAKE_BUILD_TYPE)
+ set(CMAKE_BUILD_TYPE Release)
+endif()
+
+set(CMAKE_CXX_FLAGS_DEBUG "-g")
+set(CMAKE_CXX_FLAGS_RELEASE "-O3")
+
+list (APPEND CMAKE_MODULE_PATH "${openmp_SOURCE_DIR}/cmake-modules")
+
+option(MKL_MT "MKL with multithreading support" ON)
+
+if(MKL_MT)
+ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+ set(OPENMP_FLAGS "-fopenmp")
+ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+ set(OPENMP_FLAGS "-fopenmp")
+ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+ set(OPENMP_FLAGS "-openmp")
+ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
+ set(OPENMP_FLAGS "-fopenmp")
+ endif()
+endif()
+
+find_package(MKL REQUIRED)
+
+if (MKL_FOUND)
+ set(LAPACK_INCLUDE_DIRS ${MKL_INCLUDE_DIRS})
+ if(MKL_MT)
+ set(LAPACK_LIBRARIES_MT
+ mkl_intel_lp64 mkl_core mkl_gnu_thread
+ gomp pthread m dl mkl_blacs_intelmpi_lp64
+ mkl_scalapack_lp64
+ CACHE PATH "LAPACK_LIBS"
+ )
+ else()
+ set(LAPACK_LIBRARIES
+ mkl_intel_lp64 mkl_core mkl_sequential
+ pthread m dl mkl_blacs_intelmpi_lp64
+ mkl_scalapack_lp64
+ CACHE PATH "LAPACK_LIBS"
+ )
+ endif()
+ link_directories(${MKL_LIB_DIR})
+endif()
+
+find_package(Boost REQUIRED COMPONENTS program_options)
+
+include(CheckCXXSymbolExists)
+CHECK_CXX_SYMBOL_EXISTS(MPI_Finalize mpi.h COMPILER_HAS_MPI)
+
+if(COMPILER_HAS_MPI)
+ add_library(MPI::MPI_C INTERFACE IMPORTED)
+else()
+ find_package(MPI REQUIRED)
+ if(MPI_FOUND)
+ SET(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
+ endif()
+endif()
+
+add_library(mkl_common INTERFACE)
+target_include_directories(mkl_common
+ INTERFACE
+ ${LAPACK_INCLUDE_DIRS})
+
+target_link_libraries(mkl_common
+ INTERFACE
+ ${LAPACK_LIBRARIES}
+ Boost::program_options)
+
+if(MKL_MT)
+add_library(mkl_common_mt INTERFACE)
+ target_include_directories(mkl_common_mt
+ INTERFACE
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+ $<INSTALL_INTERFACE:include>
+ ${LAPACK_INCLUDE_DIRS})
+
+ target_link_libraries(mkl_common_mt
+ INTERFACE
+ ${LAPACK_LIBRARIES_MT}
+ Boost::program_options)
+endif()
+
+add_library(mkl_common_with_mpi INTERFACE)
+
+if(COMPILER_HAS_MPI)
+ target_link_libraries(mkl_common_with_mpi
+ INTERFACE
+ mkl_common
+ MPI::MPI_C)
+else()
+ target_link_libraries(mkl_common_with_mpi
+ INTERFACE
+ mkl_common
+ ${MPI_CXX_LIBRARIES})
+endif()
+
+SET(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
+
+ADD_SUBDIRECTORY(0_hello_world)
+ADD_SUBDIRECTORY(1_integral)
+
+include(Dart)
+include(CPack)
+
+
diff --git a/README.md b/README.md
index 41a36296074b04c94c760cf1c2a130f0342dd7cb..52d622620567f5ecd7ebeebc1a49b6a2186332c3 100644
--- a/README.md
+++ b/README.md
@@ -2,6 +2,10 @@
OpenMP test codes
+## To run with hpxMP on JUWELS
+
+1. load the modules
+
```bash
module load GCC
module load ParaStationMPI/5.2.2-1
@@ -10,4 +14,16 @@ module load CMake/3.14.0
module load Boost/1.69.0-Python-2.7.16
module load CUDA/10.1.105
module load hwloc/2.1.0
+module load imkl
+```
+2. compile the code with normal openMP flags by GCC or others
+
+```bash
+gcc -fopenmp my_application.c -o my_application
+```
+
+3. run the application with corresponding hpxMP library
+
+```bash
+LD_PRELOAD=/p/project/cslai/wu/source/hpxMP/hpxMP-0.3.0/build/gcc_juwels/libhpxmp.so ./my_application
```
diff --git a/cmake-modules/FindBLIS.cmake b/cmake-modules/FindBLIS.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..6a7b48e947ea5f5e1b51cdab4a2bc544deb14d12
--- /dev/null
+++ b/cmake-modules/FindBLIS.cmake
@@ -0,0 +1,64 @@
+# - Find BLIS
+# Find the BLIS libraries
+# BLIS: BLAS-like Library Instantiation Software Framework
+# Github: https://github.com/flame/blis
+# BLIS_FOUND : True if BLIS_INCUDE_DIR are found
+# BLIS_INCLUDE_DIR : where to find blis.h, etc.
+# BLIS_INCLUDE_DIRS : set when BLIS_INCLUDE_DIR found
+# BLIS_LIBRARIES : the library to link against.
+
+set(BLIS_ROOT "/usr/local" CACHE PATH "Folder containing BLIS libraries")
+
+if (NOT BLIS_ROOT AND DEFINED ENV{BLIS_ROOT})
+ set(BLIS_ROOT $ENV{BLIS_ROOT} CACHE PATH "Folder containing BLIS")
+elseif (NOT BLIS_ROOT AND DEFINED ENV{BLISROOT})
+ set(BLIS_ROOT $ENV{BISLROOT} CACHE PATH "Folder containing BLIS")
+endif()
+
+find_path(BLIS_INCLUDE_DIR
+ NAMES
+ blis.h
+ PATHS
+ ${BLIS_ROOT}/include/blis
+)
+
+set(_BLIS_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+
+if(WIN32)
+ if(BLIS_STATIC)
+ set(CMAKE_FIND_LIBRARY_SUFFIXES .lib)
+ else()
+ set(CMAKE_FIND_LIBRARY_SUFFIXES _dll.lib)
+ endif()
+else()
+ if(BLIS_STATIC)
+ set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
+ else()
+ if(APPLE)
+ set(CMAKE_FIND_LIBRARY_SUFFIXES .dylib)
+ else()
+ set(CMAKE_FIND_LIBRARY_SUFFIXES .so)
+ endif()
+ endif()
+endif()
+
+find_library(BLIS_LIBRARY blis
+ PATHS ${BLIS_ROOT}/lib
+)
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(BLIS
+ DEFAULT_MSG
+ BLIS_LIBRARY
+ BLIS_INCLUDE_DIR
+)
+
+if(BLIS_FOUND)
+ set(BLIS_INCLUDE_DIRS ${BLIS_INCLUDE_DIR})
+ set(BLIS_LIBRARIES ${BLIS_LIBRARY})
+endif()
+
+
+
+
diff --git a/cmake-modules/FindElemental.cmake b/cmake-modules/FindElemental.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..288f931637d8b3edf75399959e03e5102f67f3b2
--- /dev/null
+++ b/cmake-modules/FindElemental.cmake
@@ -0,0 +1,82 @@
+# - Find Elemental
+# Find the Elemental libraries
+# Elemental: Distributed-memory, arbitrary-precision, dense and sparse-direct
+# linear algebra, conic optimization, and lattice reduction
+# Github: https://github.com/elemental/Elemental
+# Elemental_FOUND : True if Elemental_INCUDE_DIR are found
+# Elemental_INCLUDE_DIR : where to find elemental.h, etc.
+# Elemental_INCLUDE_DIRS : set when Elemental_INCLUDE_DIR found
+# Elemental_LIBRARIES : the library to link against.
+# PMRRR_LIBRARY : PMRRR_LIBRARY to link
+
+set(Elemental_ROOT "/usr/local" CACHE PATH "Folder containing Elemental libraries")
+
+if (NOT Elemental_ROOT AND DEFINED ENV{Elemental_ROOT})
+ set(Elemental_ROOT $ENV{Elemental_ROOT} CACHE PATH "Folder containing Elemental")
+elseif (NOT Elemental_ROOT AND DEFINED ENV{ElementalROOT})
+ set(Elemental_ROOT $ENV{ElementalROOT} CACHE PATH "Folder containing Elemental")
+endif()
+
+find_file(Elemental_basic_header
+ NAMES
+ elemental.hpp
+ PATHS
+ ${Elemental_ROOT}/include
+)
+
+if(Elemental_basic_header)
+
+ find_path(Elemental_INCLUDE_DIR
+ NAMES
+ elemental.hpp
+ PATHS
+ ${Elemental_ROOT}/include
+ )
+
+ find_library(Elemental_LIBRARY libelemental.a
+ PATHS ${Elemental_ROOT}/lib
+ )
+
+ find_library(PMRRR_LIBRARY libpmrrr.a
+ PATHS ${Elemental_ROOT}/lib
+ )
+
+ set(LOW_VERISON true)
+
+else()
+
+ find_path(Elemental_INCLUDE_DIR
+ NAMES
+ El.hpp
+ PATHS
+ ${Elemental_ROOT}/include
+ )
+
+ find_library(Elemental_LIBRARY libEl.so
+ PATHS ${Elemental_ROOT}/lib
+ )
+
+ find_library(PMRRR_LIBRARY libpmrrr.so
+ PATHS ${Elemental_ROOT}/lib
+ )
+
+ set(LOW_VERISON false)
+
+endif()
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(Elemental
+ DEFAULT_MSG
+ Elemental_LIBRARY
+ Elemental_INCLUDE_DIR
+)
+
+if(Elemental_FOUND)
+ set(Elemental_INCLUDE_DIRS ${Elemental_INCLUDE_DIR})
+ set(Elemental_LIBRARIES ${Elemental_LIBRARY})
+ set(Elemental_IF_LOW_VERSION ${LOW_VERISON})
+endif()
+
+
+
diff --git a/cmake-modules/FindHWLOC.cmake b/cmake-modules/FindHWLOC.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..690b3d105965cd0940d14c73eb493fb65422861a
--- /dev/null
+++ b/cmake-modules/FindHWLOC.cmake
@@ -0,0 +1,195 @@
+# FindHwloc
+# ----------
+#
+#
+# You may declare HWLOC_ROOT environment variable to tell where
+# your hwloc library is installed.
+#
+# Once done this will define::
+#
+# Hwloc_FOUND - True if hwloc was found
+# Hwloc_INCLUDE_DIRS - include directories for hwloc
+# Hwloc_LIBRARIES - link against these libraries to use hwloc
+# Hwloc_VERSION - version
+# Hwloc_CFLAGS - include directories as compiler flags
+# Hwloc_LDLFAGS - link paths and libs as compiler flags
+#
+
+if(WIN32)
+ find_path(Hwloc_INCLUDE_DIR
+ NAMES
+ hwloc.h
+ PATHS
+ ENV "PROGRAMFILES(X86)"
+ ENV HWLOC_ROOT
+ PATH_SUFFIXES
+ include
+ )
+
+ find_library(Hwloc_LIBRARY
+ NAMES
+ libhwloc.lib
+ PATHS
+ ENV "PROGRAMFILES(X86)"
+ ENV HWLOC_ROOT
+ PATH_SUFFIXES
+ lib
+ )
+
+ #
+ # Check if the found library can be used to linking
+ #
+ SET (_TEST_SOURCE "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/linktest.c")
+ FILE (WRITE "${_TEST_SOURCE}"
+ "
+ #include <hwloc.h>
+ int main()
+ {
+ hwloc_topology_t topology;
+ int nbcores;
+ hwloc_topology_init(&topology);
+ hwloc_topology_load(topology);
+ nbcores = hwloc_get_nbobjs_by_type(topology, HWLOC_OBJ_CORE);
+ hwloc_topology_destroy(topology);
+ return 0;
+ }
+ "
+ )
+
+ TRY_COMPILE(_LINK_SUCCESS ${CMAKE_BINARY_DIR} "${_TEST_SOURCE}"
+ CMAKE_FLAGS
+ "-DINCLUDE_DIRECTORIES:STRING=${Hwloc_INCLUDE_DIR}"
+ CMAKE_FLAGS
+ "-DLINK_LIBRARIES:STRING=${Hwloc_LIBRARY}"
+ )
+
+ IF(NOT _LINK_SUCCESS)
+ if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+ message(STATUS "You are building 64bit target.")
+ ELSE()
+ message(STATUS "You are building 32bit code. If you like to build x64 use e.g. -G 'Visual Studio 12 Win64' generator." )
+ ENDIF()
+ message(FATAL_ERROR "Library found, but linking test program failed.")
+ ENDIF()
+
+ #
+ # Resolve version if some compiled binary found...
+ #
+ find_program(HWLOC_INFO_EXECUTABLE
+ NAMES
+ hwloc-info
+ PATHS
+ ENV HWLOC_ROOT
+ PATH_SUFFIXES
+ bin
+ )
+
+ if(HWLOC_INFO_EXECUTABLE)
+ execute_process(
+ COMMAND ${HWLOC_INFO_EXECUTABLE} "--version"
+ OUTPUT_VARIABLE HWLOC_VERSION_LINE
+ OUTPUT_STRIP_TRAILING_WHITESPACE
+ )
+ string(REGEX MATCH "([0-9]+.[0-9]+)$"
+ Hwloc_VERSION "${HWLOC_VERSION_LINE}")
+ unset(HWLOC_VERSION_LINE)
+ endif()
+
+ #
+ # All good
+ #
+
+ set(Hwloc_LIBRARIES ${Hwloc_LIBRARY})
+ set(Hwloc_INCLUDE_DIRS ${Hwloc_INCLUDE_DIR})
+
+ include(FindPackageHandleStandardArgs)
+ find_package_handle_standard_args(
+ Hwloc
+ FOUND_VAR Hwloc_FOUND
+ REQUIRED_VARS Hwloc_LIBRARY Hwloc_INCLUDE_DIR
+ VERSION_VAR Hwloc_VERSION)
+
+ mark_as_advanced(
+ Hwloc_INCLUDE_DIR
+ Hwloc_LIBRARY)
+
+ foreach(arg ${Hwloc_INCLUDE_DIRS})
+ set(Hwloc_CFLAGS "${Hwloc_CFLAGS} /I${arg}")
+ endforeach()
+
+ set(Hwloc_LDFLAGS "${Hwloc_LIBRARY}")
+
+else()
+
+ if(CMAKE_CROSSCOMPILING)
+
+ find_path(Hwloc_INCLUDE_DIRS
+ NAMES
+ hwloc.h
+ PATHS
+ ENV HWLOC_ROOT
+ )
+
+ find_library(Hwloc_LIBRARIES
+ NAMES
+ hwloc
+ PATHS
+ ENV HWLOC_ROOT
+ )
+
+ if(Hwloc_INCLUDE_DIRS AND Hwloc_LIBRARIES)
+ message(WARNING "HWLOC library found using find_library() - cannot determine version. Assuming 1.7.0")
+ set(Hwloc_FOUND 1)
+ set(Hwloc_VERSION "1.7.0")
+ endif()
+
+ else() # Find with pkgconfig for non-crosscompile builds
+
+ find_package(PkgConfig)
+
+ if(HWLOC_ROOT)
+ set(ENV{PKG_CONFIG_PATH} "${HWLOC_ROOT}/lib/pkgconfig")
+ else()
+ foreach(PREFIX ${CMAKE_PREFIX_PATH})
+ set(PKG_CONFIG_PATH "${PKG_CONFIG_PATH}:${PREFIX}/lib/pkgconfig")
+ endforeach()
+ set(ENV{PKG_CONFIG_PATH} "${PKG_CONFIG_PATH}:$ENV{PKG_CONFIG_PATH}")
+ endif()
+
+ if(hwloc_FIND_REQUIRED)
+ set(_hwloc_OPTS "REQUIRED")
+ elseif(hwloc_FIND_QUIETLY)
+ set(_hwloc_OPTS "QUIET")
+ else()
+ set(_hwloc_output 1)
+ endif()
+
+ if(hwloc_FIND_VERSION)
+ if(hwloc_FIND_VERSION_EXACT)
+ pkg_check_modules(Hwloc ${_hwloc_OPTS} hwloc=${hwloc_FIND_VERSION})
+ else()
+ pkg_check_modules(Hwloc ${_hwloc_OPTS} hwloc>=${hwloc_FIND_VERSION})
+ endif()
+ else()
+ pkg_check_modules(Hwloc ${_hwloc_OPTS} hwloc)
+ endif()
+
+ if(Hwloc_FOUND)
+ include(FindPackageHandleStandardArgs)
+ find_package_handle_standard_args(Hwloc DEFAULT_MSG Hwloc_LIBRARIES)
+
+ if(NOT ${Hwloc_VERSION} VERSION_LESS 1.7.0)
+ set(Hwloc_GL_FOUND 1)
+ endif()
+
+ if(_hwloc_output)
+ message(STATUS
+ "Found hwloc ${Hwloc_VERSION} in ${Hwloc_INCLUDE_DIRS}:${Hwloc_LIBRARIES}")
+ endif()
+ endif()
+
+ endif() # cross-compile else
+
+endif()
+
+
diff --git a/cmake-modules/FindMKL.cmake b/cmake-modules/FindMKL.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..26c8654cea6ccf712cd483a4a2608f6a3e865bf7
--- /dev/null
+++ b/cmake-modules/FindMKL.cmake
@@ -0,0 +1,177 @@
+# - Find Intel MKL
+# Find the MKL libraries
+#
+# Options:
+#
+# MKL_ROOT : place to search for MKL
+# $ENV{MKL_ROOT} : " " " " " " "
+# $ENV{MKLROOT} : " " " " " " "
+# MKL_STATIC : use static linking
+# MKL_MULTI_THREADED : use multi-threading
+# MKL_SDL : Single Dynamic Library interface
+# MKL_32 : search for ia32 instead of intel64
+# MKL_VERBOSE : display info about found heders/libs/etc
+#
+# This module defines the following variables:
+#
+# MKL_FOUND : True if MKL_INCLUDE_DIR are found
+# MKL_INCLUDE_DIR : where to find mkl.h, etc.
+# MKL_INCLUDE_DIRS : set when MKL_INCLUDE_DIR found
+# MKL_LIB_DIR : the dir where the mkl libs are found
+# MKL_LIBRARIES : the library to link against.
+# MKL_LIBRARIES_MT : the library to link against.
+
+macro(MKL_MESSAGE string)
+ if (MKL_VERBOSE)
+ message("MKL: " ${string})
+ endif()
+endmacro()
+
+include(FindPackageHandleStandardArgs)
+
+set(INTEL_ROOT "/opt/intel" CACHE PATH "Folder containing intel libs")
+if (NOT MKL_ROOT AND DEFINED ENV{MKL_ROOT})
+ set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder containing MKL")
+elseif (NOT MKL_ROOT AND DEFINED ENV{MKLROOT})
+ set(MKL_ROOT $ENV{MKLROOT} CACHE PATH "Folder containing MKL")
+elseif (NOT MKL_ROOT)
+ set(MKL_ROOT ${INTEL_ROOT}/mkl CACHE PATH "Folder containing MKL")
+endif()
+
+MKL_MESSAGE("Using MKL_ROOT ${MKL_ROOT}")
+
+# Find include dir
+find_path(MKL_INCLUDE_DIR mkl.h
+ PATHS ${MKL_ROOT}/include)
+
+MKL_MESSAGE("MKL_INCLUDE_DIR is ${MKL_INCLUDE_DIR}")
+
+# Find include directory
+# There is no include folder under linux
+if(WIN32)
+ find_path(INTEL_INCLUDE_DIR omp.h
+ PATHS ${INTEL_ROOT}/include)
+ set(MKL_INCLUDE_DIR ${MKL_INCLUDE_DIR} ${INTEL_INCLUDE_DIR})
+endif()
+
+# Find libraries
+
+# Handle suffix
+set(_MKL_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
+
+if(WIN32)
+ if(MKL_STATIC)
+ set(CMAKE_FIND_LIBRARY_SUFFIXES .lib)
+ else()
+ set(CMAKE_FIND_LIBRARY_SUFFIXES _dll.lib)
+ endif()
+else()
+ if(MKL_STATIC)
+ set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
+ else()
+ if(APPLE)
+ set(CMAKE_FIND_LIBRARY_SUFFIXES .dylib)
+ else()
+ set(CMAKE_FIND_LIBRARY_SUFFIXES .so)
+ endif()
+ endif()
+endif()
+
+if(UNIX AND NOT APPLE)
+ if(MKL_32)
+ set(MKL_SUBDIR ia32)
+ else()
+ set(MKL_SUBDIR intel64)
+ MKL_MESSAGE("MKL sub dir is ${MKL_SUBDIR}")
+ endif()
+endif()
+
+set(MKL_LIB_DIR "${MKL_ROOT}/lib/${MKL_SUBDIR}" CACHE INTERNAL "lib path")
+MKL_MESSAGE("MKL_LIB_DIR sub dir is ${MKL_LIB_DIR}")
+
+# MKL is composed by four layers: Interface, Threading, Computational and RTL
+
+if(MKL_SDL)
+ find_library(MKL_LIBRARY mkl_rt
+ PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+
+ set(MKL_MINIMAL_LIBRARY ${MKL_LIBRARY})
+else()
+ ######################### Interface layer #######################
+ if(WIN32)
+ set(MKL_INTERFACE_LIBNAME mkl_intel_c)
+ else()
+ set(MKL_INTERFACE_LIBNAME mkl_intel_ilp64)
+ endif()
+
+ find_library(MKL_INTERFACE_LIBRARY ${MKL_INTERFACE_LIBNAME}
+ PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+ MKL_MESSAGE("MKL_INTERFACE_LIBRARY is ${MKL_INTERFACE_LIBRARY}")
+
+ ######################## Threading layer ########################
+ find_library(MKL_THREADING_LIBRARY mkl_sequential
+ PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+ find_library(MKL_THREADING_LIBRARY_MT mkl_gnu_thread
+ PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+
+ ####################### Computational layer #####################
+ find_library(MKL_CORE_LIBRARY mkl_core
+ PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+ find_library(MKL_FFT_LIBRARY mkl_cdft_core
+ PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+ find_library(MKL_SCALAPACK_LIBRARY mkl_scalapack_ilp64
+ PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+ find_library(MKL_BLACS_LIBRARY mkl_blacs_intelmpi_ilp64
+ PATHS ${MKL_ROOT}/lib/${MKL_SUBDIR}/)
+
+
+ MKL_MESSAGE("MKL_CORE_LIBRARY is ${MKL_CORE_LIBRARY}")
+ MKL_MESSAGE("MKL_FFT_LIBRARY is ${MKL_FFT_LIBRARY}")
+ MKL_MESSAGE("MKL_SCALAPACK_LIBRARY is ${MKL_SCALAPACK_LIBRARY}")
+ MKL_MESSAGE("MKL_BLACS_LIBRARY is ${MKL_BLACS_LIBRARY}")
+
+ ############################ RTL layer ##########################
+ if(WIN32)
+ set(MKL_RTL_LIBNAME libiomp5md)
+ else()
+ set(MKL_RTL_LIBNAME libiomp5)
+ endif()
+ find_library(MKL_RTL_LIBRARY ${MKL_RTL_LIBNAME}
+ PATHS ${INTEL_RTL_ROOT}/lib)
+ MKL_MESSAGE("MKL_RTL_LIBRARY is ${MKL_RTL_LIBRARY}")
+
+ set(MKL_LIBRARY
+ ${MKL_INTERFACE_LIBRARY}
+ ${MKL_CORE_LIBRARY}
+ ${MKL_THREADING_LIBRARY}
+ ${MKL_FFT_LIBRARY}
+ ${MKL_SCALAPACK_LIBRARY}
+ )
+ set(MKL_MINIMAL_LIBRARY ${MKL_INTERFACE_LIBRARY} ${MKL_THREADING_LIBRARY} ${MKL_CORE_LIBRARY})
+ if (MKL_RTL_LIBRARY)
+ set(MKL_LIBRARY ${MKL_LIBRARY} ${MKL_RTL_LIBRARY})
+ set(MKL_MINIMAL_LIBRARY ${MKL_MINIMAL_LIBRARY} ${MKL_RTL_LIBRARY})
+ endif()
+endif()
+
+set(CMAKE_FIND_LIBRARY_SUFFIXES ${_MKL_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
+
+find_package_handle_standard_args(MKL DEFAULT_MSG
+ MKL_INCLUDE_DIR MKL_LIBRARY MKL_MINIMAL_LIBRARY)
+
+if(MKL_FOUND)
+ set(MKL_INCLUDE_DIRS ${MKL_INCLUDE_DIR})
+ set(MKL_LIBRARIES ${MKL_LIBRARY})
+ set(MKL_MINIMAL_LIBRARIES ${MKL_LIBRARY})
+ set(MKL_LIBRARIES_MT
+ ${MKL_INTERFACE_LIBRARY}
+ ${MKL_CORE_LIBRARY}
+ ${MKL_THREADING_LIBRARY_MT}
+ ${MKL_FFT_LIBRARY}
+ ${MKL_SCALAPACK_LIBRARY}
+ ${MKL_BLACS_LIBRARY}
+ )
+ MKL_MESSAGE("MKL_LIBRARIES_MT are ${MKL_LIBRARIES_MT}")
+endif()
+
+
diff --git a/cmake-modules/FindPAPI.cmake b/cmake-modules/FindPAPI.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..03f351fdb41549ef59171a5cbed7ee02102c181a
--- /dev/null
+++ b/cmake-modules/FindPAPI.cmake
@@ -0,0 +1,42 @@
+# - Find PAPI
+# Find the PAPI libraries
+# PAPI: Performance Application Programming Interface
+# Website: https://icl.utk.edu/papi/index.html
+# PAPI_FOUND : True if PAPI_INCUDE_DIR are found
+# PAPI_INCLUDE_DIR : where to find papi.h, etc.
+# PAPI_INCLUDE_DIRS : set when PAPU_INCLUDE_DIR found
+# PAPI_LIBRARIES : the library to link against.
+
+set(PAPI_ROOT "/usr/local" CACHE PATH "Folder containing PAPI libraries")
+
+if (NOT PAPI_ROOT AND DEFINED ENV{PAPI_ROOT})
+ set(PAPI_ROOT $ENV{PAPI_ROOT} CACHE PATH "Folder containing PAPI")
+elseif (NOT PAPI_ROOT AND DEFINED ENV{PAPIROOT})
+ set(PAPI_ROOT $ENV{PAPIROOT} CACHE PATH "Folder containing PAPI")
+endif()
+
+find_path(PAPI_INCLUDE_DIR
+ NAMES
+ papi.h
+ PATHS
+ ${PAPI_ROOT}/include
+)
+
+find_library(PAPI_LIBRARY libpapi.so libpapi.a papi
+ PATHS ${PAPI_ROOT}/lib
+)
+
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(PAPI
+ DEFAULT_MSG
+ PAPI_LIBRARY
+ PAPI_INCLUDE_DIR
+)
+
+if(PAPI_FOUND)
+ set(PAPI_INCLUDE_DIRS ${PAPI_INCLUDE_DIR})
+ set(PAPI_LIBRARIES ${PAPI_LIBRARY})
+endif()
+
+