Resolve "implement lazy data preprocessing"

Closes #292 (closed)

mlair/data_handler/default_data_handler.py

@@ -8,6 +8,7 @@ import gc
 import logging
 import os
 import pickle
+import dill
 import shutil
 from functools import reduce
 from typing import Tuple, Union, List
@@ -86,7 +87,7 @@ class DefaultDataHandler(AbstractDataHandler):
             data = {"X": self._X, "Y": self._Y, "X_extreme": self._X_extreme, "Y_extreme": self._Y_extreme}
             data = self._force_dask_computation(data)
             with open(self._save_file, "wb") as f:
-                pickle.dump(data, f)
+                dill.dump(data, f)
             logging.debug(f"save pickle data to {self._save_file}")
             self._reset_data()
 
@@ -101,7 +102,7 @@ class DefaultDataHandler(AbstractDataHandler):
     def _load(self):
         try:
             with open(self._save_file, "rb") as f:
-                data = pickle.load(f)
+                data = dill.load(f)
             logging.debug(f"load pickle data from {self._save_file}")
             self._X, self._Y = data["X"], data["Y"]
             self._X_extreme, self._Y_extreme = data["X_extreme"], data["Y_extreme"]