diff --git a/mlair/data_handler/data_handler_kz_filter.py b/mlair/data_handler/data_handler_kz_filter.py index 1ff1a36f61670383e1f3bf9314045e799be7394d..1f2c63e58c7eaab645f074ac953d2f05d8ba09fd 100644 --- a/mlair/data_handler/data_handler_kz_filter.py +++ b/mlair/data_handler/data_handler_kz_filter.py @@ -8,6 +8,7 @@ import numpy as np import pandas as pd import xarray as xr from typing import List, Union +from functools import partial from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation from mlair.data_handler import DefaultDataHandler @@ -83,6 +84,14 @@ class DataHandlerKzFilterSingleStation(DataHandlerSingleStation): return self.history.transpose(self.time_dim, self.window_dim, self.iter_dim, self.target_dim, self.filter_dim).copy() + def _create_lazy_data(self): + return [self._data, self.meta, self.input_data, self.target_data, self.cutoff_period, self.cutoff_period_days] + + def _extract_lazy(self, lazy_data): + _data, self.meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days = lazy_data + f_prep = partial(self._slice_prep, start=self.start, end=self.end) + self._data, self.input_data, self.target_data = list(map(f_prep, [_data, _input_data, _target_data])) + class DataHandlerKzFilter(DefaultDataHandler): """Data handler using kz filtered data.""" diff --git a/mlair/data_handler/data_handler_mixed_sampling.py b/mlair/data_handler/data_handler_mixed_sampling.py index acb62df9ce1fd9b61889c0b97973c953e9bda1ff..b359a26df539e95736650a9d8fc5116ea68db8ee 100644 --- a/mlair/data_handler/data_handler_mixed_sampling.py +++ b/mlair/data_handler/data_handler_mixed_sampling.py @@ -12,9 +12,6 @@ import inspect from typing import Callable import datetime as dt from typing import Any -import os -import dill -import logging from functools import partial import numpy as np @@ -158,9 +155,6 @@ class DataHandlerMixedSamplingWithFilterSingleStation(DataHandlerMixedSamplingSi limit=self.interpolation_limit[ind]) return data - def _create_lazy_data(self): - return [self._data, self.meta, self.input_data, self.target_data, self.cutoff_period, self.cutoff_period_days] - def _extract_lazy(self, lazy_data): _data, self.meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days = lazy_data start_inp, end_inp = self.update_start_end(0) diff --git a/mlair/data_handler/data_handler_single_station.py b/mlair/data_handler/data_handler_single_station.py index a8c6ea2e021251b72a9699426dc179d82fcb4e6d..0497bee0ae6b6a72301181ef5453dd40f479e5af 100644 --- a/mlair/data_handler/data_handler_single_station.py +++ b/mlair/data_handler/data_handler_single_station.py @@ -61,11 +61,11 @@ class DataHandlerSingleStation(AbstractDataHandler): interpolation_method: Union[str, Tuple[str]] = DEFAULT_INTERPOLATION_METHOD, overwrite_local_data: bool = False, transformation=None, store_data_locally: bool = True, min_length: int = 0, start=None, end=None, variables=None, data_origin: Dict = None, - lazy_loading: bool = False, **kwargs): + lazy_preprocessing: bool = False, **kwargs): super().__init__() self.station = helpers.to_list(station) self.path = self.setup_data_path(data_path, sampling) - self.lazy = lazy_loading + self.lazy = lazy_preprocessing self.lazy_path = None if self.lazy is True: self.lazy_path = os.path.join(data_path, "lazy_data", self.__class__.__name__) @@ -252,9 +252,7 @@ class DataHandlerSingleStation(AbstractDataHandler): with open(filename, "rb") as pickle_file: lazy_data = dill.load(pickle_file) self._extract_lazy(lazy_data) - logging.info("<<<loaded lazy file") except FileNotFoundError: - logging.info(">>>could not load lazy file") self.make_input_target() def _extract_lazy(self, lazy_data):