From 02a051fedde8124043997770882d1984c51ad323 Mon Sep 17 00:00:00 2001
From: leufen1 <l.leufen@fz-juelich.de>
Date: Mon, 22 Feb 2021 16:23:53 +0100
Subject: [PATCH] use 16 kernels in maximum for parallel preprocessing
---
mlair/run_modules/pre_processing.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/mlair/run_modules/pre_processing.py b/mlair/run_modules/pre_processing.py
index 0ca355c9..cdf195e7 100644
--- a/mlair/run_modules/pre_processing.py
+++ b/mlair/run_modules/pre_processing.py
@@ -265,7 +265,8 @@ class PreProcessing(RunEnvironment):
if multiprocessing.cpu_count() > 1: # parallel solution
logging.info("use parallel validate station approach")
- pool = multiprocessing.Pool(psutil.cpu_count(logical=False)) # use only physical cpus
+ pool = multiprocessing.Pool(
+ min([psutil.cpu_count(logical=False), len(set_stations), 16])) # use only physical cpus
logging.info(f"running {getattr(pool, '_processes')} processes in parallel")
output = [
pool.apply_async(f_proc, args=(data_handler, station, set_name, store_processed_data), kwds=kwargs)
--
GitLab