Skip to content
Snippets Groups Projects
Commit a6cd5f5c authored by leufen1's avatar leufen1
Browse files

all filter data handlers can now return either filter portions as dimension or distinct branches

parent e872a388
No related branches found
No related tags found
5 merge requests!319add all changes of dev into release v1.4.0 branch,!318Resolve "release v1.4.0",!317enabled window_lead_time=1,!295Resolve "data handler FIR filter",!259Draft: Resolve "WRF-Datahandler should inherit from SingleStationDatahandler"
...@@ -4,6 +4,8 @@ __date__ = '2020-11-05' ...@@ -4,6 +4,8 @@ __date__ = '2020-11-05'
from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilterSingleStation, \ from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilterSingleStation, \
DataHandlerFirFilterSingleStation, DataHandlerFilterSingleStation, DataHandlerClimateFirFilterSingleStation DataHandlerFirFilterSingleStation, DataHandlerFilterSingleStation, DataHandlerClimateFirFilterSingleStation
from mlair.data_handler.data_handler_with_filter import DataHandlerClimateFirFilter, DataHandlerFirFilter, \
DataHandlerKzFilter
from mlair.data_handler import DefaultDataHandler from mlair.data_handler import DefaultDataHandler
from mlair import helpers from mlair import helpers
from mlair.helpers import remove_items from mlair.helpers import remove_items
...@@ -176,11 +178,12 @@ class DataHandlerMixedSamplingWithKzFilterSingleStation(DataHandlerMixedSampling ...@@ -176,11 +178,12 @@ class DataHandlerMixedSamplingWithKzFilterSingleStation(DataHandlerMixedSampling
return int(self.kz_filter_length[0] * np.sqrt(self.kz_filter_iter[0]) * 2) return int(self.kz_filter_length[0] * np.sqrt(self.kz_filter_iter[0]) * 2)
def _extract_lazy(self, lazy_data): def _extract_lazy(self, lazy_data):
_data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days = lazy_data _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days, \
self.filter_dim_order = lazy_data
super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data)) super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
class DataHandlerMixedSamplingWithKzFilter(DefaultDataHandler): class DataHandlerMixedSamplingWithKzFilter(DataHandlerKzFilter):
"""Data handler using mixed sampling for input and target. Inputs are temporal filtered.""" """Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
data_handler = DataHandlerMixedSamplingWithKzFilterSingleStation data_handler = DataHandlerMixedSamplingWithKzFilterSingleStation
...@@ -199,7 +202,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin ...@@ -199,7 +202,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
return max(self.filter_order) return max(self.filter_order)
def _extract_lazy(self, lazy_data): def _extract_lazy(self, lazy_data):
_data, _meta, _input_data, _target_data, self.fir_coeff = lazy_data _data, _meta, _input_data, _target_data, self.fir_coeff, self.filter_dim_order = lazy_data
super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data)) super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
@staticmethod @staticmethod
...@@ -214,7 +217,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin ...@@ -214,7 +217,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
raise ValueError(f"Unknown sampling rate {sampling}. Only daily and hourly resolution is supported.") raise ValueError(f"Unknown sampling rate {sampling}. Only daily and hourly resolution is supported.")
class DataHandlerMixedSamplingWithFirFilter(DefaultDataHandler): class DataHandlerMixedSamplingWithFirFilter(DataHandlerFirFilter):
"""Data handler using mixed sampling for input and target. Inputs are temporal filtered.""" """Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
data_handler = DataHandlerMixedSamplingWithFirFilterSingleStation data_handler = DataHandlerMixedSamplingWithFirFilterSingleStation
...@@ -236,7 +239,8 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed ...@@ -236,7 +239,8 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def _extract_lazy(self, lazy_data): def _extract_lazy(self, lazy_data):
_data, _meta, _input_data, _target_data, self.climate_filter_coeff, self.apriori, self.all_apriori = lazy_data _data, _meta, _input_data, _target_data, self.climate_filter_coeff, self.apriori, self.all_apriori, \
self.filter_dim_order = lazy_data
DataHandlerMixedSamplingWithFilterSingleStation._extract_lazy(self, (_data, _meta, _input_data, _target_data)) DataHandlerMixedSamplingWithFilterSingleStation._extract_lazy(self, (_data, _meta, _input_data, _target_data))
@staticmethod @staticmethod
...@@ -251,7 +255,7 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed ...@@ -251,7 +255,7 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed
raise ValueError(f"Unknown sampling rate {sampling}. Only daily and hourly resolution is supported.") raise ValueError(f"Unknown sampling rate {sampling}. Only daily and hourly resolution is supported.")
class DataHandlerMixedSamplingWithClimateFirFilter(DefaultDataHandler): class DataHandlerMixedSamplingWithClimateFirFilter(DataHandlerClimateFirFilter):
"""Data handler using mixed sampling for input and target. Inputs are temporal filtered.""" """Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
data_handler = DataHandlerMixedSamplingWithClimateFirFilterSingleStation data_handler = DataHandlerMixedSamplingWithClimateFirFilterSingleStation
......
...@@ -47,6 +47,7 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation): ...@@ -47,6 +47,7 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation):
def __init__(self, *args, filter_dim=DEFAULT_FILTER_DIM, **kwargs): def __init__(self, *args, filter_dim=DEFAULT_FILTER_DIM, **kwargs):
# self.original_data = None # ToDo: implement here something to store unfiltered data # self.original_data = None # ToDo: implement here something to store unfiltered data
self.filter_dim = filter_dim self.filter_dim = filter_dim
self.filter_dim_order = None
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def setup_transformation(self, transformation: Union[None, dict, Tuple]) -> Tuple[Optional[dict], Optional[dict]]: def setup_transformation(self, transformation: Union[None, dict, Tuple]) -> Tuple[Optional[dict], Optional[dict]]:
...@@ -104,6 +105,38 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation): ...@@ -104,6 +105,38 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation):
self._data, self.input_data, self.target_data = list(map(f_prep, [_data, _input_data, _target_data])) self._data, self.input_data, self.target_data = list(map(f_prep, [_data, _input_data, _target_data]))
class DataHandlerFilter(DefaultDataHandler):
"""Data handler using FIR filtered data."""
data_handler = DataHandlerFilterSingleStation
data_handler_transformation = DataHandlerFilterSingleStation
_requirements = data_handler.requirements()
def __init__(self, *args, use_filter_branches=False, **kwargs):
self.use_filter_branches = use_filter_branches
super().__init__(*args, **kwargs)
@classmethod
def own_args(cls, *args):
"""Return all arguments (including kwonlyargs)."""
super_own_args = DefaultDataHandler.own_args(*args)
arg_spec = inspect.getfullargspec(cls)
list_of_args = arg_spec.args + arg_spec.kwonlyargs + super_own_args
return remove_items(list_of_args, ["self"] + list(args))
def get_X_original(self):
if self.use_filter_branches is True:
X = []
for data in self._collection:
X_total = data.get_X()
filter_dim = data.filter_dim
for filter_name in data.filter_dim_order:
X.append(X_total.sel({filter_dim: filter_name}, drop=True))
return X
else:
return super().get_X_original()
class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation): class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
"""Data handler for a single station to be used by a superior data handler. Inputs are FIR filtered.""" """Data handler for a single station to be used by a superior data handler. Inputs are FIR filtered."""
...@@ -203,17 +236,18 @@ class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation): ...@@ -203,17 +236,18 @@ class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
band_num += 1 band_num += 1
if self._add_unfiltered: if self._add_unfiltered:
index.append("unfiltered") index.append("unfiltered")
self.filter_dim_order = index
return pd.Index(index, name=self.filter_dim) return pd.Index(index, name=self.filter_dim)
def _create_lazy_data(self): def _create_lazy_data(self):
return [self._data, self.meta, self.input_data, self.target_data, self.fir_coeff] return [self._data, self.meta, self.input_data, self.target_data, self.fir_coeff, self.filter_dim_order]
def _extract_lazy(self, lazy_data): def _extract_lazy(self, lazy_data):
_data, _meta, _input_data, _target_data, self.fir_coeff = lazy_data _data, _meta, _input_data, _target_data, self.fir_coeff, self.filter_dim_order = lazy_data
super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data)) super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
class DataHandlerFirFilter(DefaultDataHandler): class DataHandlerFirFilter(DataHandlerFilter):
"""Data handler using FIR filtered data.""" """Data handler using FIR filtered data."""
data_handler = DataHandlerFirFilterSingleStation data_handler = DataHandlerFirFilterSingleStation
...@@ -262,17 +296,20 @@ class DataHandlerKzFilterSingleStation(DataHandlerFilterSingleStation): ...@@ -262,17 +296,20 @@ class DataHandlerKzFilterSingleStation(DataHandlerFilterSingleStation):
f = lambda x: int(np.round(x)) if x >= 10 else np.round(x, 1) f = lambda x: int(np.round(x)) if x >= 10 else np.round(x, 1)
index = list(map(f, index.tolist())) index = list(map(f, index.tolist()))
index = list(map(lambda x: str(x) + "d", index)) + ["res"] index = list(map(lambda x: str(x) + "d", index)) + ["res"]
self.filter_dim_order = index
return pd.Index(index, name=self.filter_dim) return pd.Index(index, name=self.filter_dim)
def _create_lazy_data(self): def _create_lazy_data(self):
return [self._data, self.meta, self.input_data, self.target_data, self.cutoff_period, self.cutoff_period_days] return [self._data, self.meta, self.input_data, self.target_data, self.cutoff_period, self.cutoff_period_days,
self.filter_dim_order]
def _extract_lazy(self, lazy_data): def _extract_lazy(self, lazy_data):
_data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days = lazy_data _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days, \
self.filter_dim_order = lazy_data
super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data)) super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
class DataHandlerKzFilter(DefaultDataHandler): class DataHandlerKzFilter(DataHandlerFilter):
"""Data handler using kz filtered data.""" """Data handler using kz filtered data."""
data_handler = DataHandlerKzFilterSingleStation data_handler = DataHandlerKzFilterSingleStation
...@@ -372,14 +409,16 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation ...@@ -372,14 +409,16 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
index = list(map(lambda x: str(x) + "d", index)) + ["res"] index = list(map(lambda x: str(x) + "d", index)) + ["res"]
if self._add_unfiltered: if self._add_unfiltered:
index.append("unfiltered") index.append("unfiltered")
self.filter_dim_order = index
return pd.Index(index, name=self.filter_dim) return pd.Index(index, name=self.filter_dim)
def _create_lazy_data(self): def _create_lazy_data(self):
return [self._data, self.meta, self.input_data, self.target_data, self.climate_filter_coeff, return [self._data, self.meta, self.input_data, self.target_data, self.climate_filter_coeff,
self.apriori, self.all_apriori] self.apriori, self.all_apriori, self.filter_dim_order]
def _extract_lazy(self, lazy_data): def _extract_lazy(self, lazy_data):
_data, _meta, _input_data, _target_data, self.climate_filter_coeff, self.apriori, self.all_apriori = lazy_data _data, _meta, _input_data, _target_data, self.climate_filter_coeff, self.apriori, self.all_apriori, \
self.filter_dim_order = lazy_data
DataHandlerSingleStation._extract_lazy(self, (_data, _meta, _input_data, _target_data)) DataHandlerSingleStation._extract_lazy(self, (_data, _meta, _input_data, _target_data))
@staticmethod @staticmethod
...@@ -440,10 +479,19 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation ...@@ -440,10 +479,19 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
self._transformation = (opts_input, opts_target) self._transformation = (opts_input, opts_target)
class DataHandlerClimateFirFilter(DefaultDataHandler): class DataHandlerClimateFirFilter(DataHandlerFilter):
"""Data handler using climatic adjusted FIR filtered data.""" """Data handler using climatic adjusted FIR filtered data."""
data_handler = DataHandlerClimateFirFilterSingleStation data_handler = DataHandlerClimateFirFilterSingleStation
data_handler_transformation = DataHandlerClimateFirFilterSingleStation data_handler_transformation = DataHandlerClimateFirFilterSingleStation
_requirements = data_handler.requirements() _requirements = data_handler.requirements()
_store_attributes = data_handler.store_attributes() _store_attributes = data_handler.store_attributes()
# def get_X_original(self):
# X = []
# for data in self._collection:
# X_total = data.get_X()
# filter_dim = data.filter_dim
# for filter in data.filter_dim_order:
# X.append(X_total.sel({filter_dim: filter}, drop=True))
# return X
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment