Skip to content
Snippets Groups Projects
Commit fe40c9e1 authored by leufen1's avatar leufen1
Browse files

new data handler that combines kz filter and mixed sampling types

parent f3c6026c
No related branches found
No related tags found
3 merge requests!192include Develop,!191Resolve "release v1.1.0",!182Resolve "Mixed sampling types"
Pipeline #51765 passed
...@@ -24,7 +24,7 @@ class DataHandlerKzFilterSingleStation(DataHandlerSingleStation): ...@@ -24,7 +24,7 @@ class DataHandlerKzFilterSingleStation(DataHandlerSingleStation):
_requirements = remove_items(inspect.getfullargspec(DataHandlerSingleStation).args, ["self", "station"]) _requirements = remove_items(inspect.getfullargspec(DataHandlerSingleStation).args, ["self", "station"])
def __init__(self, *args, kz_filter_length, kz_filter_iter, **kwargs): def __init__(self, *args, kz_filter_length, kz_filter_iter, **kwargs):
assert kwargs.get("sampling") == "hourly" # This data handler requires hourly data resolution self._check_sampling(**kwargs)
kz_filter_length = to_list(kz_filter_length) kz_filter_length = to_list(kz_filter_length)
kz_filter_iter = to_list(kz_filter_iter) kz_filter_iter = to_list(kz_filter_iter)
# self.original_data = None # ToDo: implement here something to store unfiltered data # self.original_data = None # ToDo: implement here something to store unfiltered data
...@@ -34,6 +34,9 @@ class DataHandlerKzFilterSingleStation(DataHandlerSingleStation): ...@@ -34,6 +34,9 @@ class DataHandlerKzFilterSingleStation(DataHandlerSingleStation):
self.cutoff_period_days = None self.cutoff_period_days = None
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
def _check_sampling(self, **kwargs):
assert kwargs.get("sampling") == "hourly" # This data handler requires hourly data resolution
def setup_samples(self): def setup_samples(self):
""" """
Setup samples. This method prepares and creates samples X, and labels Y. Setup samples. This method prepares and creates samples X, and labels Y.
......
...@@ -2,6 +2,7 @@ __author__ = 'Lukas Leufen' ...@@ -2,6 +2,7 @@ __author__ = 'Lukas Leufen'
__date__ = '2020-11-05' __date__ = '2020-11-05'
from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
from mlair.data_handler.data_handler_kz_filter import DataHandlerKzFilterSingleStation
from mlair.data_handler import DefaultDataHandler from mlair.data_handler import DefaultDataHandler
from mlair.configuration import path_config from mlair.configuration import path_config
from mlair import helpers from mlair import helpers
...@@ -59,3 +60,38 @@ class DataHandlerMixedSampling(DefaultDataHandler): ...@@ -59,3 +60,38 @@ class DataHandlerMixedSampling(DefaultDataHandler):
data_handler = DataHandlerMixedSamplingSingleStation data_handler = DataHandlerMixedSamplingSingleStation
data_handler_transformation = DataHandlerMixedSamplingSingleStation data_handler_transformation = DataHandlerMixedSamplingSingleStation
_requirements = data_handler.requirements() _requirements = data_handler.requirements()
class DataHandlerMixedSamplingWithFilterSingleStation(DataHandlerMixedSamplingSingleStation,
DataHandlerKzFilterSingleStation):
_requirements1 = DataHandlerKzFilterSingleStation.requirements()
_requirements2 = DataHandlerMixedSamplingSingleStation.requirements()
_requirements = list(set(_requirements1 + _requirements2))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def _check_sampling(self, **kwargs):
assert kwargs.get("sampling") == ("hourly", "daily")
def setup_samples(self):
"""
Setup samples. This method prepares and creates samples X, and labels Y.
A KZ filter is applied on the input data that has hourly resolution. Lables Y are provided as aggregated values
with daily resolution.
"""
self._data = list(map(self.load_and_interpolate, [0, 1])) # load input (0) and target (1) data
self.set_inputs_and_targets()
self.apply_kz_filter()
if self.do_transformation is True:
self.call_transform()
self.make_samples()
class DataHandlerMixedSamplingWithFilter(DefaultDataHandler):
"""Data handler using mixed sampling for input and target. Inputs are temporal filtered."""
data_handler = DataHandlerMixedSamplingWithFilterSingleStation
data_handler_transformation = DataHandlerMixedSamplingWithFilterSingleStation
_requirements = data_handler.requirements()
...@@ -4,7 +4,7 @@ __date__ = '2019-11-14' ...@@ -4,7 +4,7 @@ __date__ = '2019-11-14'
import argparse import argparse
from mlair.workflows import DefaultWorkflow from mlair.workflows import DefaultWorkflow
from mlair.data_handler.data_handler_mixed_sampling import DataHandlerMixedSampling from mlair.data_handler.data_handler_mixed_sampling import DataHandlerMixedSampling, DataHandlerMixedSamplingWithFilter
def main(parser_args): def main(parser_args):
...@@ -12,7 +12,9 @@ def main(parser_args): ...@@ -12,7 +12,9 @@ def main(parser_args):
sampling_inputs="hourly", sampling_inputs="hourly",
window_history_size=72, window_history_size=72,
**parser_args.__dict__, **parser_args.__dict__,
data_handler=DataHandlerMixedSampling, data_handler=DataHandlerMixedSampling, # WithFilter,
kz_filter_length=[365 * 24, 20 * 24],
kz_filter_iter=[3, 5],
start="2006-01-01", start="2006-01-01",
train_start="2006-01-01", train_start="2006-01-01",
end="2011-12-31", end="2011-12-31",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment