diff --git a/mlair/data_handler/data_handler_mixed_sampling.py b/mlair/data_handler/data_handler_mixed_sampling.py
index a389445e8af8db0559fd1bf8f3445475cb62082e..79e40424549a7c2e16c08f618e649f08984919fa 100644
--- a/mlair/data_handler/data_handler_mixed_sampling.py
+++ b/mlair/data_handler/data_handler_mixed_sampling.py
@@ -4,23 +4,17 @@ __date__ = '2020-11-05'
from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
from mlair.data_handler.data_handler_with_filter import DataHandlerFirFilterSingleStation, \
DataHandlerFilterSingleStation, DataHandlerClimateFirFilterSingleStation
-# from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilterSingleStation
from mlair.data_handler.data_handler_with_filter import DataHandlerClimateFirFilter, DataHandlerFirFilter
-# from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilter
from mlair.data_handler import DefaultDataHandler
from mlair import helpers
-from mlair.helpers import remove_items
from mlair.configuration.defaults import DEFAULT_SAMPLING, DEFAULT_INTERPOLATION_LIMIT, DEFAULT_INTERPOLATION_METHOD
from mlair.helpers.filter import filter_width_kzf
import copy
-import inspect
-from typing import Callable
import datetime as dt
from typing import Any
from functools import partial
-import numpy as np
import pandas as pd
import xarray as xr
@@ -113,6 +107,16 @@ class DataHandlerMixedSamplingWithFilterSingleStation(DataHandlerMixedSamplingSi
def _check_sampling(self, **kwargs):
assert kwargs.get("sampling") == ("hourly", "daily")
+ def apply_filter(self):
+ raise NotImplementedError
+
+ def create_filter_index(self) -> pd.Index:
+ """Create name for filter dimension."""
+ raise NotImplementedError
+
+ def _create_lazy_data(self):
+ raise NotImplementedError
+
def make_input_target(self):
"""
A FIR filter is applied on the input data that has hourly resolution. Lables Y are provided as aggregated values
@@ -160,36 +164,10 @@ class DataHandlerMixedSamplingWithFilterSingleStation(DataHandlerMixedSamplingSi
self.input_data = self._slice_prep(_input_data, start_inp, end_inp)
self.target_data = self._slice_prep(_target_data, self.start, self.end)
-#
-# class DataHandlerMixedSamplingWithKzFilterSingleStation(DataHandlerMixedSamplingWithFilterSingleStation,
-# DataHandlerKzFilterSingleStation):
-# _requirements1 = DataHandlerKzFilterSingleStation.requirements()
-# _requirements2 = DataHandlerMixedSamplingWithFilterSingleStation.requirements()
-# _requirements = list(set(_requirements1 + _requirements2))
-#
-# def estimate_filter_width(self):
-# """
-# f = 0.5 / (len * sqrt(itr)) -> T = 1 / f
-# :return:
-# """
-# return int(self.kz_filter_length[0] * np.sqrt(self.kz_filter_iter[0]) * 2)
-#
-# def _extract_lazy(self, lazy_data):
-# _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days, \
-# self.filter_dim_order = lazy_data
-# super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
-#
-#
-# class DataHandlerMixedSamplingWithKzFilter(DataHandlerKzFilter):
-# """Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
-#
-# data_handler = DataHandlerMixedSamplingWithKzFilterSingleStation
-# data_handler_transformation = DataHandlerMixedSamplingWithKzFilterSingleStation
-# _requirements = list(set(data_handler.requirements() + DataHandlerKzFilter.requirements()))
-
class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplingWithFilterSingleStation,
DataHandlerFirFilterSingleStation):
+
_requirements1 = DataHandlerFirFilterSingleStation.requirements()
_requirements2 = DataHandlerMixedSamplingWithFilterSingleStation.requirements()
_requirements = list(set(_requirements1 + _requirements2))
@@ -204,9 +182,17 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
else:
return max(self.filter_order)
+ def apply_filter(self):
+ DataHandlerFirFilterSingleStation.apply_filter()
+
+ def create_filter_index(self, add_unfiltered_index=True) -> pd.Index:
+ return DataHandlerFirFilterSingleStation.create_filter_index(self, add_unfiltered_index=add_unfiltered_index)
+
def _extract_lazy(self, lazy_data):
- _data, _meta, _input_data, _target_data, self.fir_coeff, self.filter_dim_order = lazy_data
- super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
+ DataHandlerFirFilterSingleStation._extract_lazy(self, lazy_data)
+
+ def _create_lazy_data(self):
+ return DataHandlerFirFilterSingleStation._create_lazy_data(self)
@staticmethod
def _get_fs(**kwargs):
diff --git a/mlair/data_handler/data_handler_with_filter.py b/mlair/data_handler/data_handler_with_filter.py
index 1c72ed6e6aa1d6069d71f5a54dd0d9deadf173d3..93118bafa77004fbabf2020309bcad99c3c86e34 100644
--- a/mlair/data_handler/data_handler_with_filter.py
+++ b/mlair/data_handler/data_handler_with_filter.py
@@ -3,7 +3,6 @@
__author__ = 'Lukas Leufen'
__date__ = '2020-08-26'
-import inspect
import copy
import numpy as np
import pandas as pd
@@ -13,8 +12,7 @@ from functools import partial
import logging
from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
from mlair.data_handler import DefaultDataHandler
-from mlair.helpers import remove_items, to_list, TimeTrackingWrapper, statistics
-from mlair.helpers.filter import KolmogorovZurbenkoFilterMovingWindow as KZFilter
+from mlair.helpers import to_list, TimeTrackingWrapper, statistics
from mlair.helpers.filter import FIRFilter, ClimateFIRFilter, omega_null_kzf
# define a more general date type for type hinting
@@ -223,7 +221,7 @@ class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
def _extract_lazy(self, lazy_data):
_data, _meta, _input_data, _target_data, self.fir_coeff, self.filter_dim_order = lazy_data
- super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
+ super()._extract_lazy((_data, _meta, _input_data, _target_data))
def transform(self, data_in, dim: Union[str, int] = 0, inverse: bool = False, opts=None,
transformation_dim=None):
@@ -310,68 +308,6 @@ class DataHandlerFirFilter(DataHandlerFilter):
data_handler_transformation = DataHandlerFirFilterSingleStation
_requirements = list(set(data_handler.requirements() + DataHandlerFilter.requirements()))
-#
-# class DataHandlerKzFilterSingleStation(DataHandlerFilterSingleStation):
-# """Data handler for a single station to be used by a superior data handler. Inputs are kz filtered."""
-#
-# _requirements = DataHandlerFilterSingleStation.requirements()
-# _hash = DataHandlerFilterSingleStation._hash + ["kz_filter_length", "kz_filter_iter"]
-#
-# def __init__(self, *args, kz_filter_length, kz_filter_iter, **kwargs):
-# self._check_sampling(**kwargs)
-# # self.original_data = None # ToDo: implement here something to store unfiltered data
-# self.kz_filter_length = to_list(kz_filter_length)
-# self.kz_filter_iter = to_list(kz_filter_iter)
-# self.cutoff_period = None
-# self.cutoff_period_days = None
-# super().__init__(*args, **kwargs)
-#
-# @TimeTrackingWrapper
-# def apply_filter(self):
-# """Apply kolmogorov zurbenko filter only on inputs."""
-# kz = KZFilter(self.input_data, wl=self.kz_filter_length, itr=self.kz_filter_iter, filter_dim=self.time_dim)
-# filtered_data: List[xr.DataArray] = kz.run()
-# self.cutoff_period = kz.period_null()
-# self.cutoff_period_days = kz.period_null_days()
-# self.input_data = xr.concat(filtered_data, pd.Index(self.create_filter_index(), name=self.filter_dim))
-# # this is just a code snippet to check the results of the kz filter
-# # import matplotlib
-# # matplotlib.use("TkAgg")
-# # import matplotlib.pyplot as plt
-# # self.input_data.sel(filter="74d", variables="temp", Stations="DEBW107").plot()
-# # self.input_data.sel(variables="temp", Stations="DEBW107").plot.line(hue="filter")
-#
-# def create_filter_index(self) -> pd.Index:
-# """
-# Round cut off periods in days and append 'res' for residuum index.
-#
-# Round small numbers (<10) to single decimal, and higher numbers to int. Transform as list of str and append
-# 'res' for residuum index.
-# """
-# index = np.round(self.cutoff_period_days, 1)
-# f = lambda x: int(np.round(x)) if x >= 10 else np.round(x, 1)
-# index = list(map(f, index.tolist()))
-# index = list(map(lambda x: str(x) + "d", index)) + ["res"]
-# self.filter_dim_order = index
-# return pd.Index(index, name=self.filter_dim)
-#
-# def _create_lazy_data(self):
-# return [self._data, self.meta, self.input_data, self.target_data, self.cutoff_period, self.cutoff_period_days,
-# self.filter_dim_order]
-#
-# def _extract_lazy(self, lazy_data):
-# _data, _meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days, \
-# self.filter_dim_order = lazy_data
-# super(__class__, self)._extract_lazy((_data, _meta, _input_data, _target_data))
-#
-#
-# class DataHandlerKzFilter(DataHandlerFilter):
-# """Data handler using kz filtered data."""
-#
-# data_handler = DataHandlerKzFilterSingleStation
-# data_handler_transformation = DataHandlerKzFilterSingleStation
-# _requirements = list(set(data_handler.requirements() + DataHandlerFilter.requirements()))
-#
class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation):
"""
diff --git a/mlair/helpers/helpers.py b/mlair/helpers/helpers.py
index 679f5a28fc564d56cd6f3794ee8fe8e1877b2b4c..77be3e2c8edcd43207ac40f67370067826be064b 100644
--- a/mlair/helpers/helpers.py
+++ b/mlair/helpers/helpers.py
@@ -4,7 +4,6 @@ __date__ = '2019-10-21'
import inspect
import math
-import sys
import numpy as np
import xarray as xr
diff --git a/test/test_data_handler/test_data_handler_mixed_sampling.py b/test/test_data_handler/test_data_handler_mixed_sampling.py
index 04ce8dc17fe18b1fddd38e42b8d6a464ea831e63..6a4f5da262e298ad0eb6993b3dd2adf76a742d6e 100644
--- a/test/test_data_handler/test_data_handler_mixed_sampling.py
+++ b/test/test_data_handler/test_data_handler_mixed_sampling.py
@@ -6,21 +6,15 @@ from mlair.data_handler.data_handler_mixed_sampling import DataHandlerMixedSampl
DataHandlerMixedSamplingWithFirFilterSingleStation, DataHandlerMixedSamplingWithFirFilter, \
DataHandlerFirFilterSingleStation, DataHandlerMixedSamplingWithClimateFirFilterSingleStation, \
DataHandlerMixedSamplingWithClimateFirFilter
-# from mlair.data_handler.data_handler_mixed_sampling import DataHandlerMixedSamplingWithKzFilter, \
-# DataHandlerMixedSamplingWithKzFilterSingleStation, DataHandlerSeparationOfScales, \
-# DataHandlerSeparationOfScalesSingleStation
from mlair.data_handler.data_handler_with_filter import DataHandlerFilter, DataHandlerFilterSingleStation, \
DataHandlerClimateFirFilterSingleStation
-# from mlair.data_handler.data_handler_with_filter import DataHandlerKzFilter, DataHandlerKzFilterSingleStation
from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
from mlair.data_handler.default_data_handler import DefaultDataHandler
-from mlair.helpers import remove_items
from mlair.configuration.defaults import DEFAULT_INTERPOLATION_METHOD
from mlair.helpers.testing import get_all_args
import pytest
import mock
-import inspect
class TestDataHandlerMixedSampling:
@@ -101,53 +95,18 @@ class TestDataHandlerMixedSamplingSingleStation:
def test_setup_data_path(self):
pass
-#
-# class TestDataHandlerMixedSamplingWithKzFilter:
-#
-# def test_data_handler(self):
-# obj = object.__new__(DataHandlerMixedSamplingWithKzFilter)
-# assert obj.data_handler.__qualname__ == DataHandlerMixedSamplingWithKzFilterSingleStation.__qualname__
-#
-# def test_data_handler_transformation(self):
-# obj = object.__new__(DataHandlerMixedSamplingWithKzFilter)
-# assert obj.data_handler_transformation.__qualname__ == DataHandlerMixedSamplingWithKzFilterSingleStation.__qualname__
-#
-# def test_requirements(self):
-#
-# reqs = get_all_args(DataHandlerFilter)
-# obj = object.__new__(DataHandlerMixedSamplingWithKzFilter)
-# assert sorted(obj.own_args()) == reqs
-# reqs = get_all_args(DataHandlerFilter, DefaultDataHandler, DataHandlerKzFilterSingleStation,
-# DataHandlerFilterSingleStation, DataHandlerSingleStation, remove=["self", "id_class"])
-# assert sorted(obj._requirements) == reqs
-# assert sorted(obj.requirements()) == reqs
-#
-#
-# class TestDataHandlerMixedSamplingWithKzFilterSingleStation:
-#
-# def test_requirements(self):
-#
-# reqs = get_all_args(DataHandlerMixedSamplingWithKzFilterSingleStation)
-# obj = object.__new__(DataHandlerMixedSamplingWithKzFilterSingleStation)
-# assert reqs == ["self"]
-# assert sorted(obj.own_args()) == reqs
-# reqs = get_all_args(DataHandlerKzFilterSingleStation, DataHandlerFilterSingleStation, DataHandlerSingleStation,
-# remove="self")
-# assert sorted(obj._requirements) == reqs
-# assert sorted(obj.requirements()) == reqs
-#
-#
-# class TestDataHandlerMixedSamplingWithFilterSingleStation:
-#
-# def test_requirements(self):
-#
-# reqs = get_all_args(DataHandlerMixedSamplingWithKzFilterSingleStation)
-# obj = object.__new__(DataHandlerMixedSamplingWithFilterSingleStation)
-# assert reqs == ["self"]
-# assert sorted(obj.own_args()) == reqs
-# reqs = get_all_args(DataHandlerFilterSingleStation, DataHandlerSingleStation, remove="self")
-# assert sorted(obj._requirements) == reqs
-# assert sorted(obj.requirements()) == reqs
+
+class TestDataHandlerMixedSamplingWithFilterSingleStation:
+
+ def test_requirements(self):
+
+ reqs = get_all_args(DataHandlerMixedSamplingWithFilterSingleStation)
+ obj = object.__new__(DataHandlerMixedSamplingWithFilterSingleStation)
+ assert reqs == ["self"]
+ assert sorted(obj.own_args()) == reqs
+ reqs = get_all_args(DataHandlerFilterSingleStation, DataHandlerSingleStation, remove="self")
+ assert sorted(obj._requirements) == reqs
+ assert sorted(obj.requirements()) == reqs
class TestDataHandlerMixedSamplingWithFirFilter: