all filter data handlers can now return either filter portions as dimension or distinct branches

mlair/data_handler/data_handler_mixed_sampling.py

@@ -4,6 +4,8 @@ __date__ = '2020-11-05'
 from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
 from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilterSingleStation, \
     DataHandlerFirFilterSingleStation, DataHandlerFilterSingleStation, DataHandlerClimateFirFilterSingleStation
+from mlair.data_handler.data_handler_with_filter import DataHandlerClimateFirFilter, DataHandlerFirFilter, \
+    DataHandlerKzFilter
 from mlair.data_handler import DefaultDataHandler
 from mlair import helpers
 from mlair.helpers import remove_items
@@ -176,11 +178,12 @@ class DataHandlerMixedSamplingWithKzFilterSingleStation(DataHandlerMixedSampling
         return int(self.kz_filter_length[0] * np.sqrt(self.kz_filter_iter[0]) * 2)
 
     def _extract_lazy(self, lazy_data):
-        _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days = lazy_data
+        _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days, \
+        self.filter_dim_order = lazy_data
         super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
 
 
-class DataHandlerMixedSamplingWithKzFilter(DefaultDataHandler):
+class DataHandlerMixedSamplingWithKzFilter(DataHandlerKzFilter):
     """Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
 
     data_handler = DataHandlerMixedSamplingWithKzFilterSingleStation
@@ -199,7 +202,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
         return max(self.filter_order)
 
     def _extract_lazy(self, lazy_data):
-        _data, _meta, _input_data, _target_data, self.fir_coeff = lazy_data
+        _data, _meta, _input_data, _target_data, self.fir_coeff, self.filter_dim_order = lazy_data
         super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
 
     @staticmethod
@@ -214,7 +217,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
             raise ValueError(f"Unknown sampling rate {sampling}. Only daily and hourly resolution is supported.")
 
 
-class DataHandlerMixedSamplingWithFirFilter(DefaultDataHandler):
+class DataHandlerMixedSamplingWithFirFilter(DataHandlerFirFilter):
     """Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
 
     data_handler = DataHandlerMixedSamplingWithFirFilterSingleStation
@@ -236,7 +239,8 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed
         super().__init__(*args, **kwargs)
 
     def _extract_lazy(self, lazy_data):
-        _data, _meta, _input_data, _target_data, self.climate_filter_coeff, self.apriori, self.all_apriori = lazy_data
+        _data, _meta, _input_data, _target_data, self.climate_filter_coeff, self.apriori, self.all_apriori, \
+        self.filter_dim_order = lazy_data
         DataHandlerMixedSamplingWithFilterSingleStation._extract_lazy(self, (_data, _meta, _input_data, _target_data))
 
     @staticmethod
@@ -251,7 +255,7 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed
             raise ValueError(f"Unknown sampling rate {sampling}. Only daily and hourly resolution is supported.")
 
 
-class DataHandlerMixedSamplingWithClimateFirFilter(DefaultDataHandler):
+class DataHandlerMixedSamplingWithClimateFirFilter(DataHandlerClimateFirFilter):
     """Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
 
     data_handler = DataHandlerMixedSamplingWithClimateFirFilterSingleStation

mlair/data_handler/data_handler_with_filter.py

@@ -47,6 +47,7 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation):
     def __init__(self, *args, filter_dim=DEFAULT_FILTER_DIM, **kwargs):
         # self.original_data = None  # ToDo: implement here something to store unfiltered data
         self.filter_dim = filter_dim
+        self.filter_dim_order = None
         super().__init__(*args, **kwargs)
 
     def setup_transformation(self, transformation: Union[None, dict, Tuple]) -> Tuple[Optional[dict], Optional[dict]]:
@@ -104,6 +105,38 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation):
         self._data, self.input_data, self.target_data = list(map(f_prep, [_data, _input_data, _target_data]))
 
 
+class DataHandlerFilter(DefaultDataHandler):
+    """Data handler using FIR filtered data."""
+
+    data_handler = DataHandlerFilterSingleStation
+    data_handler_transformation = DataHandlerFilterSingleStation
+    _requirements = data_handler.requirements()
+
+    def __init__(self, *args, use_filter_branches=False, **kwargs):
+        self.use_filter_branches = use_filter_branches
+        super().__init__(*args, **kwargs)
+
+    @classmethod
+    def own_args(cls, *args):
+        """Return all arguments (including kwonlyargs)."""
+        super_own_args = DefaultDataHandler.own_args(*args)
+        arg_spec = inspect.getfullargspec(cls)
+        list_of_args = arg_spec.args + arg_spec.kwonlyargs + super_own_args
+        return remove_items(list_of_args, ["self"] + list(args))
+
+    def get_X_original(self):
+        if self.use_filter_branches is True:
+            X = []
+            for data in self._collection:
+                X_total = data.get_X()
+                filter_dim = data.filter_dim
+                for filter_name in data.filter_dim_order:
+                    X.append(X_total.sel({filter_dim: filter_name}, drop=True))
+            return X
+        else:
+            return super().get_X_original()
+
+
 class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
     """Data handler for a single station to be used by a superior data handler. Inputs are FIR filtered."""
 
@@ -203,17 +236,18 @@ class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
                 band_num += 1
         if self._add_unfiltered:
             index.append("unfiltered")
+        self.filter_dim_order = index
         return pd.Index(index, name=self.filter_dim)
 
     def _create_lazy_data(self):
-        return [self._data, self.meta, self.input_data, self.target_data, self.fir_coeff]
+        return [self._data, self.meta, self.input_data, self.target_data, self.fir_coeff, self.filter_dim_order]
 
     def _extract_lazy(self, lazy_data):
-        _data, _meta, _input_data, _target_data, self.fir_coeff = lazy_data
+        _data, _meta, _input_data, _target_data, self.fir_coeff, self.filter_dim_order = lazy_data
         super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
 
 
-class DataHandlerFirFilter(DefaultDataHandler):
+class DataHandlerFirFilter(DataHandlerFilter):
     """Data handler using FIR filtered data."""
 
     data_handler = DataHandlerFirFilterSingleStation
@@ -262,17 +296,20 @@ class DataHandlerKzFilterSingleStation(DataHandlerFilterSingleStation):
         f = lambda x: int(np.round(x)) if x >= 10 else np.round(x, 1)
         index = list(map(f, index.tolist()))
         index = list(map(lambda x: str(x) + "d", index)) + ["res"]
+        self.filter_dim_order = index
         return pd.Index(index, name=self.filter_dim)
 
     def _create_lazy_data(self):
-        return [self._data, self.meta, self.input_data, self.target_data, self.cutoff_period, self.cutoff_period_days]
+        return [self._data, self.meta, self.input_data, self.target_data, self.cutoff_period, self.cutoff_period_days,
+                self.filter_dim_order]
 
     def _extract_lazy(self, lazy_data):
-        _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days = lazy_data
+        _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days, \
+        self.filter_dim_order = lazy_data
         super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
 
 
-class DataHandlerKzFilter(DefaultDataHandler):
+class DataHandlerKzFilter(DataHandlerFilter):
     """Data handler using kz filtered data."""
 
     data_handler = DataHandlerKzFilterSingleStation
@@ -372,14 +409,16 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
         index = list(map(lambda x: str(x) + "d", index)) + ["res"]
         if self._add_unfiltered:
             index.append("unfiltered")
+        self.filter_dim_order = index
         return pd.Index(index, name=self.filter_dim)
 
     def _create_lazy_data(self):
         return [self._data, self.meta, self.input_data, self.target_data, self.climate_filter_coeff,
-                self.apriori, self.all_apriori]
+                self.apriori, self.all_apriori, self.filter_dim_order]
 
     def _extract_lazy(self, lazy_data):
-        _data, _meta, _input_data, _target_data, self.climate_filter_coeff, self.apriori, self.all_apriori = lazy_data
+        _data, _meta, _input_data, _target_data, self.climate_filter_coeff, self.apriori, self.all_apriori, \
+        self.filter_dim_order = lazy_data
         DataHandlerSingleStation._extract_lazy(self, (_data, _meta, _input_data, _target_data))
 
     @staticmethod
@@ -440,10 +479,19 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
         self._transformation = (opts_input, opts_target)
 
 
-class DataHandlerClimateFirFilter(DefaultDataHandler):
+class DataHandlerClimateFirFilter(DataHandlerFilter):
     """Data handler using climatic adjusted FIR filtered data."""
 
     data_handler = DataHandlerClimateFirFilterSingleStation
     data_handler_transformation = DataHandlerClimateFirFilterSingleStation
     _requirements = data_handler.requirements()
     _store_attributes = data_handler.store_attributes()
+
+    # def get_X_original(self):
+    #     X = []
+    #     for data in self._collection:
+    #         X_total = data.get_X()
+    #         filter_dim = data.filter_dim
+    #         for filter in data.filter_dim_order:
+    #             X.append(X_total.sel({filter_dim: filter}, drop=True))
+    #     return X