Skip to content
Snippets Groups Projects
Commit e8e8aa69 authored by Felix Kleinert's avatar Felix Kleinert
Browse files

force dask.compute in try except

parent fc6625d2
No related branches found
No related tags found
6 merge requests!319add all changes of dev into release v1.4.0 branch,!318Resolve "release v1.4.0",!283Merge latest develop into falcos issue,!264Merge develop into felix_issue287_tech-wrf-datahandler-should-inherit-from-singlestationdatahandler,!263Resolve "DefaultDataHandler: dask.compute() before dump to file",!259Draft: Resolve "WRF-Datahandler should inherit from SingleStationDatahandler"
Pipeline #62002 passed
...@@ -84,11 +84,20 @@ class DefaultDataHandler(AbstractDataHandler): ...@@ -84,11 +84,20 @@ class DefaultDataHandler(AbstractDataHandler):
if store_processed_data is True: if store_processed_data is True:
self._cleanup() if fresh_store is True else None self._cleanup() if fresh_store is True else None
data = {"X": self._X, "Y": self._Y, "X_extreme": self._X_extreme, "Y_extreme": self._Y_extreme} data = {"X": self._X, "Y": self._Y, "X_extreme": self._X_extreme, "Y_extreme": self._Y_extreme}
data = self._force_dask_computation(data)
with open(self._save_file, "wb") as f: with open(self._save_file, "wb") as f:
pickle.dump(data, f) pickle.dump(data, f)
logging.debug(f"save pickle data to {self._save_file}") logging.debug(f"save pickle data to {self._save_file}")
self._reset_data() self._reset_data()
@staticmethod
def _force_dask_computation(data):
try:
data = dask.compute(data)[0]
except:
pass
return data
def _load(self): def _load(self):
try: try:
with open(self._save_file, "rb") as f: with open(self._save_file, "rb") as f:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment