reduce parallel preprocessing to physical cpus

mlair/data_handler/default_data_handler.py

@@ -12,6 +12,7 @@ import shutil
 from functools import reduce
 from typing import Tuple, Union, List
 import multiprocessing
+import psutil
 
 import numpy as np
 import xarray as xr
@@ -270,7 +271,7 @@ class DefaultDataHandler(AbstractDataHandler):
 
         if multiprocessing.cpu_count() > 1:  # parallel solution
             logging.info("use parallel transformation approach")
-            pool = multiprocessing.Pool()
+            pool = multiprocessing.Pool(psutil.cpu_count(logical=False))  # use only physical cpus
             logging.info(f"running {getattr(pool, '_processes')} processes in parallel")
             output = [
                 pool.apply_async(f_proc, args=(cls.data_handler_transformation, station), kwds=sp_keys)

mlair/run_modules/pre_processing.py

@@ -8,6 +8,7 @@ import os
 from typing import Tuple
 import multiprocessing
 import requests
+import psutil
 
 import numpy as np
 import pandas as pd
@@ -264,7 +265,7 @@ class PreProcessing(RunEnvironment):
 
         if multiprocessing.cpu_count() > 1:  # parallel solution
             logging.info("use parallel validate station approach")
-            pool = multiprocessing.Pool()
+            pool = multiprocessing.Pool(psutil.cpu_count(logical=False))  # use only physical cpus
             logging.info(f"running {getattr(pool, '_processes')} processes in parallel")
             output = [
                 pool.apply_async(f_proc, args=(data_handler, station, set_name, store_processed_data), kwds=kwargs)