Skip to content
Snippets Groups Projects
Commit 3d35e8e7 authored by leufen1's avatar leufen1
Browse files

reduce parallel preprocessing to physical cpus

parent b38104d1
Branches
Tags
5 merge requests!264Merge develop into felix_issue287_tech-wrf-datahandler-should-inherit-from-singlestationdatahandler,!259Draft: Resolve "WRF-Datahandler should inherit from SingleStationDatahandler",!253include current develop,!252Resolve "release v1.3.0",!251Resolve "Changes required from HPC tests"
Pipeline #60689 failed
......@@ -12,6 +12,7 @@ import shutil
from functools import reduce
from typing import Tuple, Union, List
import multiprocessing
import psutil
import numpy as np
import xarray as xr
......@@ -270,7 +271,7 @@ class DefaultDataHandler(AbstractDataHandler):
if multiprocessing.cpu_count() > 1: # parallel solution
logging.info("use parallel transformation approach")
pool = multiprocessing.Pool()
pool = multiprocessing.Pool(psutil.cpu_count(logical=False)) # use only physical cpus
logging.info(f"running {getattr(pool, '_processes')} processes in parallel")
output = [
pool.apply_async(f_proc, args=(cls.data_handler_transformation, station), kwds=sp_keys)
......
......@@ -8,6 +8,7 @@ import os
from typing import Tuple
import multiprocessing
import requests
import psutil
import numpy as np
import pandas as pd
......@@ -264,7 +265,7 @@ class PreProcessing(RunEnvironment):
if multiprocessing.cpu_count() > 1: # parallel solution
logging.info("use parallel validate station approach")
pool = multiprocessing.Pool()
pool = multiprocessing.Pool(psutil.cpu_count(logical=False)) # use only physical cpus
logging.info(f"running {getattr(pool, '_processes')} processes in parallel")
output = [
pool.apply_async(f_proc, args=(data_handler, station, set_name, store_processed_data), kwds=kwargs)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment