remove code that was moved to comments before

mlair/data_handler/data_handler_mixed_sampling.py

@@ -4,23 +4,17 @@ __date__ = '2020-11-05'
 from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
 from mlair.data_handler.data_handler_with_filter import DataHandlerFirFilterSingleStation, \
     DataHandlerFilterSingleStation, DataHandlerClimateFirFilterSingleStation
-# from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilterSingleStation
 from mlair.data_handler.data_handler_with_filter import DataHandlerClimateFirFilter, DataHandlerFirFilter
-# from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilter
 from mlair.data_handler import DefaultDataHandler
 from mlair import helpers
-from mlair.helpers import remove_items
 from mlair.configuration.defaults import DEFAULT_SAMPLING, DEFAULT_INTERPOLATION_LIMIT, DEFAULT_INTERPOLATION_METHOD
 from mlair.helpers.filter import filter_width_kzf
 
 import copy
-import inspect
-from typing import Callable
 import datetime as dt
 from typing import Any
 from functools import partial
 
-import numpy as np
 import pandas as pd
 import xarray as xr
 
@@ -113,6 +107,16 @@ class DataHandlerMixedSamplingWithFilterSingleStation(DataHandlerMixedSamplingSi
     def _check_sampling(self, **kwargs):
         assert kwargs.get("sampling") == ("hourly", "daily")
 
+    def apply_filter(self):
+        raise NotImplementedError
+
+    def create_filter_index(self) -> pd.Index:
+        """Create name for filter dimension."""
+        raise NotImplementedError
+
+    def _create_lazy_data(self):
+        raise NotImplementedError
+
     def make_input_target(self):
         """
         A FIR filter is applied on the input data that has hourly resolution. Lables Y are provided as aggregated values
@@ -160,36 +164,10 @@ class DataHandlerMixedSamplingWithFilterSingleStation(DataHandlerMixedSamplingSi
         self.input_data = self._slice_prep(_input_data, start_inp, end_inp)
         self.target_data = self._slice_prep(_target_data, self.start, self.end)
 
-#
-# class DataHandlerMixedSamplingWithKzFilterSingleStation(DataHandlerMixedSamplingWithFilterSingleStation,
-#                                                         DataHandlerKzFilterSingleStation):
-#     _requirements1 = DataHandlerKzFilterSingleStation.requirements()
-#     _requirements2 = DataHandlerMixedSamplingWithFilterSingleStation.requirements()
-#     _requirements = list(set(_requirements1 + _requirements2))
-#
-#     def estimate_filter_width(self):
-#         """
-#         f = 0.5 / (len * sqrt(itr)) -> T = 1 / f
-#         :return:
-#         """
-#         return int(self.kz_filter_length[0] * np.sqrt(self.kz_filter_iter[0]) * 2)
-#
-#     def _extract_lazy(self, lazy_data):
-#         _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days, \
-#         self.filter_dim_order = lazy_data
-#         super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
-#
-#
-# class DataHandlerMixedSamplingWithKzFilter(DataHandlerKzFilter):
-#     """Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
-#
-#     data_handler = DataHandlerMixedSamplingWithKzFilterSingleStation
-#     data_handler_transformation = DataHandlerMixedSamplingWithKzFilterSingleStation
-#     _requirements = list(set(data_handler.requirements() + DataHandlerKzFilter.requirements()))
-
 
 class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplingWithFilterSingleStation,
                                                          DataHandlerFirFilterSingleStation):
+
     _requirements1 = DataHandlerFirFilterSingleStation.requirements()
     _requirements2 = DataHandlerMixedSamplingWithFilterSingleStation.requirements()
     _requirements = list(set(_requirements1 + _requirements2))
@@ -204,9 +182,17 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
         else:
             return max(self.filter_order)
 
+    def apply_filter(self):
+        DataHandlerFirFilterSingleStation.apply_filter()
+
+    def create_filter_index(self, add_unfiltered_index=True) -> pd.Index:
+        return DataHandlerFirFilterSingleStation.create_filter_index(self, add_unfiltered_index=add_unfiltered_index)
+
     def _extract_lazy(self, lazy_data):
-        _data, _meta, _input_data, _target_data, self.fir_coeff, self.filter_dim_order = lazy_data
-        super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
+        DataHandlerFirFilterSingleStation._extract_lazy(self, lazy_data)
+
+    def _create_lazy_data(self):
+        return DataHandlerFirFilterSingleStation._create_lazy_data(self)
 
     @staticmethod
     def _get_fs(**kwargs):

mlair/data_handler/data_handler_with_filter.py

@@ -3,7 +3,6 @@
 __author__ = 'Lukas Leufen'
 __date__ = '2020-08-26'
 
-import inspect
 import copy
 import numpy as np
 import pandas as pd
@@ -13,8 +12,7 @@ from functools import partial
 import logging
 from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
 from mlair.data_handler import DefaultDataHandler
-from mlair.helpers import remove_items, to_list, TimeTrackingWrapper, statistics
-from mlair.helpers.filter import KolmogorovZurbenkoFilterMovingWindow as KZFilter
+from mlair.helpers import to_list, TimeTrackingWrapper, statistics
 from mlair.helpers.filter import FIRFilter, ClimateFIRFilter, omega_null_kzf
 
 # define a more general date type for type hinting
@@ -223,7 +221,7 @@ class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
 
     def _extract_lazy(self, lazy_data):
         _data, _meta, _input_data, _target_data, self.fir_coeff, self.filter_dim_order = lazy_data
-        super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
+        super()._extract_lazy((_data, _meta, _input_data, _target_data))
 
     def transform(self, data_in, dim: Union[str, int] = 0, inverse: bool = False, opts=None,
                   transformation_dim=None):
@@ -310,68 +308,6 @@ class DataHandlerFirFilter(DataHandlerFilter):
     data_handler_transformation = DataHandlerFirFilterSingleStation
     _requirements = list(set(data_handler.requirements() + DataHandlerFilter.requirements()))
 
-#
-# class DataHandlerKzFilterSingleStation(DataHandlerFilterSingleStation):
-#     """Data handler for a single station to be used by a superior data handler. Inputs are kz filtered."""
-#
-#     _requirements = DataHandlerFilterSingleStation.requirements()
-#     _hash = DataHandlerFilterSingleStation._hash + ["kz_filter_length", "kz_filter_iter"]
-#
-#     def __init__(self, *args, kz_filter_length, kz_filter_iter, **kwargs):
-#         self._check_sampling(**kwargs)
-#         # self.original_data = None  # ToDo: implement here something to store unfiltered data
-#         self.kz_filter_length = to_list(kz_filter_length)
-#         self.kz_filter_iter = to_list(kz_filter_iter)
-#         self.cutoff_period = None
-#         self.cutoff_period_days = None
-#         super().__init__(*args, **kwargs)
-#
-#     @TimeTrackingWrapper
-#     def apply_filter(self):
-#         """Apply kolmogorov zurbenko filter only on inputs."""
-#         kz = KZFilter(self.input_data, wl=self.kz_filter_length, itr=self.kz_filter_iter, filter_dim=self.time_dim)
-#         filtered_data: List[xr.DataArray] = kz.run()
-#         self.cutoff_period = kz.period_null()
-#         self.cutoff_period_days = kz.period_null_days()
-#         self.input_data = xr.concat(filtered_data, pd.Index(self.create_filter_index(), name=self.filter_dim))
-#         # this is just a code snippet to check the results of the kz filter
-#         # import matplotlib
-#         # matplotlib.use("TkAgg")
-#         # import matplotlib.pyplot as plt
-#         # self.input_data.sel(filter="74d", variables="temp", Stations="DEBW107").plot()
-#         # self.input_data.sel(variables="temp", Stations="DEBW107").plot.line(hue="filter")
-#
-#     def create_filter_index(self) -> pd.Index:
-#         """
-#         Round cut off periods in days and append 'res' for residuum index.
-#
-#         Round small numbers (<10) to single decimal, and higher numbers to int. Transform as list of str and append
-#         'res' for residuum index.
-#         """
-#         index = np.round(self.cutoff_period_days, 1)
-#         f = lambda x: int(np.round(x)) if x >= 10 else np.round(x, 1)
-#         index = list(map(f, index.tolist()))
-#         index = list(map(lambda x: str(x) + "d", index)) + ["res"]
-#         self.filter_dim_order = index
-#         return pd.Index(index, name=self.filter_dim)
-#
-#     def _create_lazy_data(self):
-#         return [self._data, self.meta, self.input_data, self.target_data, self.cutoff_period, self.cutoff_period_days,
-#                 self.filter_dim_order]
-#
-#     def _extract_lazy(self, lazy_data):
-#         _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days, \
-#         self.filter_dim_order = lazy_data
-#         super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
-#
-#
-# class DataHandlerKzFilter(DataHandlerFilter):
-#     """Data handler using kz filtered data."""
-#
-#     data_handler = DataHandlerKzFilterSingleStation
-#     data_handler_transformation = DataHandlerKzFilterSingleStation
-#     _requirements = list(set(data_handler.requirements() + DataHandlerFilter.requirements()))
-#
 
 class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation):
     """

mlair/helpers/helpers.py

@@ -4,7 +4,6 @@ __date__ = '2019-10-21'
 
 import inspect
 import math
-import sys
 
 import numpy as np
 import xarray as xr

test/test_data_handler/test_data_handler_mixed_sampling.py

@@ -6,21 +6,15 @@ from mlair.data_handler.data_handler_mixed_sampling import DataHandlerMixedSampl
     DataHandlerMixedSamplingWithFirFilterSingleStation, DataHandlerMixedSamplingWithFirFilter, \
     DataHandlerFirFilterSingleStation, DataHandlerMixedSamplingWithClimateFirFilterSingleStation, \
     DataHandlerMixedSamplingWithClimateFirFilter
-# from mlair.data_handler.data_handler_mixed_sampling import DataHandlerMixedSamplingWithKzFilter, \
-#     DataHandlerMixedSamplingWithKzFilterSingleStation, DataHandlerSeparationOfScales, \
-#     DataHandlerSeparationOfScalesSingleStation
 from mlair.data_handler.data_handler_with_filter import DataHandlerFilter, DataHandlerFilterSingleStation, \
     DataHandlerClimateFirFilterSingleStation
-# from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilter, DataHandlerKzFilterSingleStation
 from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
 from mlair.data_handler.default_data_handler import DefaultDataHandler
-from mlair.helpers import remove_items
 from mlair.configuration.defaults import DEFAULT_INTERPOLATION_METHOD
 from mlair.helpers.testing import get_all_args
 
 import pytest
 import mock
-import inspect
 
 
 class TestDataHandlerMixedSampling:
@@ -101,53 +95,18 @@ class TestDataHandlerMixedSamplingSingleStation:
     def test_setup_data_path(self):
         pass
 
-#
-# class TestDataHandlerMixedSamplingWithKzFilter:
-#
-#     def test_data_handler(self):
-#         obj = object.__new__(DataHandlerMixedSamplingWithKzFilter)
-#         assert obj.data_handler.__qualname__ == DataHandlerMixedSamplingWithKzFilterSingleStation.__qualname__
-#
-#     def test_data_handler_transformation(self):
-#         obj = object.__new__(DataHandlerMixedSamplingWithKzFilter)
-#         assert obj.data_handler_transformation.__qualname__ == DataHandlerMixedSamplingWithKzFilterSingleStation.__qualname__
-#
-#     def test_requirements(self):
-#
-#         reqs = get_all_args(DataHandlerFilter)
-#         obj = object.__new__(DataHandlerMixedSamplingWithKzFilter)
-#         assert sorted(obj.own_args()) == reqs
-#         reqs = get_all_args(DataHandlerFilter, DefaultDataHandler, DataHandlerKzFilterSingleStation,
-#                             DataHandlerFilterSingleStation, DataHandlerSingleStation, remove=["self", "id_class"])
-#         assert sorted(obj._requirements) == reqs
-#         assert sorted(obj.requirements()) == reqs
-#
-#
-# class TestDataHandlerMixedSamplingWithKzFilterSingleStation:
-#
-#     def test_requirements(self):
-#
-#         reqs = get_all_args(DataHandlerMixedSamplingWithKzFilterSingleStation)
-#         obj = object.__new__(DataHandlerMixedSamplingWithKzFilterSingleStation)
-#         assert reqs == ["self"]
-#         assert sorted(obj.own_args()) == reqs
-#         reqs = get_all_args(DataHandlerKzFilterSingleStation, DataHandlerFilterSingleStation, DataHandlerSingleStation,
-#                             remove="self")
-#         assert sorted(obj._requirements) == reqs
-#         assert sorted(obj.requirements()) == reqs
-#
-#
-# class TestDataHandlerMixedSamplingWithFilterSingleStation:
-#
-#     def test_requirements(self):
-#
-#         reqs = get_all_args(DataHandlerMixedSamplingWithKzFilterSingleStation)
-#         obj = object.__new__(DataHandlerMixedSamplingWithFilterSingleStation)
-#         assert reqs == ["self"]
-#         assert sorted(obj.own_args()) == reqs
-#         reqs = get_all_args(DataHandlerFilterSingleStation, DataHandlerSingleStation, remove="self")
-#         assert sorted(obj._requirements) == reqs
-#         assert sorted(obj.requirements()) == reqs
+
+class TestDataHandlerMixedSamplingWithFilterSingleStation:
+
+    def test_requirements(self):
+
+        reqs = get_all_args(DataHandlerMixedSamplingWithFilterSingleStation)
+        obj = object.__new__(DataHandlerMixedSamplingWithFilterSingleStation)
+        assert reqs == ["self"]
+        assert sorted(obj.own_args()) == reqs
+        reqs = get_all_args(DataHandlerFilterSingleStation, DataHandlerSingleStation, remove="self")
+        assert sorted(obj._requirements) == reqs
+        assert sorted(obj.requirements()) == reqs
 
 
 class TestDataHandlerMixedSamplingWithFirFilter: