From ffaaf6f8dd5af179b6168a0d2ec86315ba500397 Mon Sep 17 00:00:00 2001
From: leufen1 <l.leufen@fz-juelich.de>
Date: Mon, 22 Feb 2021 16:28:45 +0100
Subject: [PATCH] use 16 kernels in maximum for parallel transformation

---
 mlair/data_handler/default_data_handler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mlair/data_handler/default_data_handler.py b/mlair/data_handler/default_data_handler.py
index d85bb4b1..52835975 100644
--- a/mlair/data_handler/default_data_handler.py
+++ b/mlair/data_handler/default_data_handler.py
@@ -271,7 +271,8 @@ class DefaultDataHandler(AbstractDataHandler):
 
         if multiprocessing.cpu_count() > 1:  # parallel solution
             logging.info("use parallel transformation approach")
-            pool = multiprocessing.Pool(psutil.cpu_count(logical=False))  # use only physical cpus
+            pool = multiprocessing.Pool(
+                min([psutil.cpu_count(logical=False), len(set_stations), 16]))  # use only physical cpus
             logging.info(f"running {getattr(pool, '_processes')} processes in parallel")
             output = [
                 pool.apply_async(f_proc, args=(cls.data_handler_transformation, station), kwds=sp_keys)
-- 
GitLab