diff --git a/mlair/data_handler/default_data_handler.py b/mlair/data_handler/default_data_handler.py index d85bb4b1233a7324ccda5247cc917f1a847b6363..52835975101f5ce6881b72b127e16c0e299dfb14 100644 --- a/mlair/data_handler/default_data_handler.py +++ b/mlair/data_handler/default_data_handler.py @@ -271,7 +271,8 @@ class DefaultDataHandler(AbstractDataHandler): if multiprocessing.cpu_count() > 1: # parallel solution logging.info("use parallel transformation approach") - pool = multiprocessing.Pool(psutil.cpu_count(logical=False)) # use only physical cpus + pool = multiprocessing.Pool( + min([psutil.cpu_count(logical=False), len(set_stations), 16])) # use only physical cpus logging.info(f"running {getattr(pool, '_processes')} processes in parallel") output = [ pool.apply_async(f_proc, args=(cls.data_handler_transformation, station), kwds=sp_keys)