diff --git a/mlair/run_modules/pre_processing.py b/mlair/run_modules/pre_processing.py index 0ca355c93a8fd2a5777f554a96cf8d1bb1528812..cdf195e705238252b117955ab1959c4177cbd17a 100644 --- a/mlair/run_modules/pre_processing.py +++ b/mlair/run_modules/pre_processing.py @@ -265,7 +265,8 @@ class PreProcessing(RunEnvironment): if multiprocessing.cpu_count() > 1: # parallel solution logging.info("use parallel validate station approach") - pool = multiprocessing.Pool(psutil.cpu_count(logical=False)) # use only physical cpus + pool = multiprocessing.Pool( + min([psutil.cpu_count(logical=False), len(set_stations), 16])) # use only physical cpus logging.info(f"running {getattr(pool, '_processes')} processes in parallel") output = [ pool.apply_async(f_proc, args=(data_handler, station, set_name, store_processed_data), kwds=kwargs)