Skip to content
Snippets Groups Projects
Commit 02a051fe authored by leufen1's avatar leufen1
Browse files

use 16 kernels in maximum for parallel preprocessing

parent 7ca31559
No related branches found
No related tags found
5 merge requests!264Merge develop into felix_issue287_tech-wrf-datahandler-should-inherit-from-singlestationdatahandler,!259Draft: Resolve "WRF-Datahandler should inherit from SingleStationDatahandler",!253include current develop,!252Resolve "release v1.3.0",!251Resolve "Changes required from HPC tests"
Pipeline #60720 passed with warnings
......@@ -265,7 +265,8 @@ class PreProcessing(RunEnvironment):
if multiprocessing.cpu_count() > 1: # parallel solution
logging.info("use parallel validate station approach")
pool = multiprocessing.Pool(psutil.cpu_count(logical=False)) # use only physical cpus
pool = multiprocessing.Pool(
min([psutil.cpu_count(logical=False), len(set_stations), 16])) # use only physical cpus
logging.info(f"running {getattr(pool, '_processes')} processes in parallel")
output = [
pool.apply_async(f_proc, args=(data_handler, station, set_name, store_processed_data), kwds=kwargs)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment