From e8e8aa69bd28828af6a5c8e018efb8dad013bd49 Mon Sep 17 00:00:00 2001
From: Felix Kleinert <f.kleinert@fz-juelich.de>
Date: Fri, 5 Mar 2021 14:02:37 +0100
Subject: [PATCH] force dask.compute in try except
---
mlair/data_handler/default_data_handler.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/mlair/data_handler/default_data_handler.py b/mlair/data_handler/default_data_handler.py
index 8553aec7..ddf276cf 100644
--- a/mlair/data_handler/default_data_handler.py
+++ b/mlair/data_handler/default_data_handler.py
@@ -84,11 +84,20 @@ class DefaultDataHandler(AbstractDataHandler):
if store_processed_data is True:
self._cleanup() if fresh_store is True else None
data = {"X": self._X, "Y": self._Y, "X_extreme": self._X_extreme, "Y_extreme": self._Y_extreme}
+ data = self._force_dask_computation(data)
with open(self._save_file, "wb") as f:
pickle.dump(data, f)
logging.debug(f"save pickle data to {self._save_file}")
self._reset_data()
+ @staticmethod
+ def _force_dask_computation(data):
+ try:
+ data = dask.compute(data)[0]
+ except:
+ pass
+ return data
+
def _load(self):
try:
with open(self._save_file, "rb") as f:
--
GitLab