Skip to content
Snippets Groups Projects
Commit 121ed8a1 authored by lukas leufen's avatar lukas leufen
Browse files

Merge branch 'lukas_issue290_bug_fix-separation-of-scales-data-handler' into 'develop'

Resolve "fix separation of scales data handler"

See merge request !272
parents 023ee744 145b3d7d
No related branches found
No related tags found
7 merge requests!319add all changes of dev into release v1.4.0 branch,!318Resolve "release v1.4.0",!283Merge latest develop into falcos issue,!279include Develop,!278Felix issue295 transformation parameters in data handler,!272Resolve "fix separation of scales data handler",!259Draft: Resolve "WRF-Datahandler should inherit from SingleStationDatahandler"
Pipeline #63914 passed
......@@ -7,7 +7,7 @@ import inspect
import numpy as np
import pandas as pd
import xarray as xr
from typing import List, Union
from typing import List, Union, Tuple, Optional
from functools import partial
from mlair.data_handler.data_handler_single_station import DataHandlerSingleStation
......@@ -37,6 +37,19 @@ class DataHandlerKzFilterSingleStation(DataHandlerSingleStation):
self.cutoff_period_days = None
super().__init__(*args, **kwargs)
def setup_transformation(self, transformation: Union[None, dict, Tuple]) -> Tuple[Optional[dict], Optional[dict]]:
"""
Adjust setup of transformation because kfz filtered data will have negative values which is not compatible with
the log transformation. Therefore, replace all log transformation methods by a default standardization. This is
only applied on input side.
"""
transformation = super(__class__, self).setup_transformation(transformation)
if transformation[0] is not None:
for k, v in transformation[0].items():
if v["method"] == "log":
transformation[0][k]["method"] = "standardise"
return transformation
def _check_sampling(self, **kwargs):
assert kwargs.get("sampling") == "hourly" # This data handler requires hourly data resolution
......
......@@ -158,7 +158,7 @@ class DataHandlerMixedSamplingWithFilterSingleStation(DataHandlerMixedSamplingSi
def _extract_lazy(self, lazy_data):
_data, self.meta, _input_data, _target_data, self.cutoff_period, self.cutoff_period_days = lazy_data
start_inp, end_inp = self.update_start_end(0)
self._data = list(map(self._slice_prep, _data, [start_inp, self.start], [end_inp, self.end]))
self._data = list(map(lambda x: self._slice_prep(_data[x], *self.update_start_end(x)), [0, 1]))
self.input_data = self._slice_prep(_input_data, start_inp, end_inp)
self.target_data = self._slice_prep(_target_data, self.start, self.end)
......
......@@ -252,7 +252,9 @@ class DataHandlerSingleStation(AbstractDataHandler):
with open(filename, "rb") as pickle_file:
lazy_data = dill.load(pickle_file)
self._extract_lazy(lazy_data)
logging.debug(f"{self.station[0]}: used lazy data")
except FileNotFoundError:
logging.debug(f"{self.station[0]}: could not use lazy data")
self.make_input_target()
def _extract_lazy(self, lazy_data):
......@@ -594,8 +596,7 @@ class DataHandlerSingleStation(AbstractDataHandler):
"""
return data.loc[{coord: slice(str(start), str(end))}]
@staticmethod
def setup_transformation(transformation: Union[None, dict, Tuple]) -> Tuple[Optional[dict], Optional[dict]]:
def setup_transformation(self, transformation: Union[None, dict, Tuple]) -> Tuple[Optional[dict], Optional[dict]]:
"""
Set up transformation by extracting all relevant information.
......
......@@ -273,7 +273,9 @@ class DefaultDataHandler(AbstractDataHandler):
if var not in transformation_dict[i].keys():
transformation_dict[i][var] = {}
opts = transformation[var]
assert transformation_dict[i][var].get("method", opts["method"]) == opts["method"]
if not transformation_dict[i][var].get("method", opts["method"]) == opts["method"]:
# data handlers with filters are allowed to change transformation method to standardise
assert hasattr(dh, "filter_dim") and opts["method"] == "standardise"
transformation_dict[i][var]["method"] = opts["method"]
for k in ["mean", "std", "min", "max"]:
old = transformation_dict[i][var].get(k, None)
......
......@@ -306,7 +306,7 @@ class PostProcessing(RunEnvironment):
try:
if ("filter" in self.test_data[0].get_X(as_numpy=False)[0].coords) and (
"PlotSeparationOfScales" in plot_list):
filter_dim = self.data_store.get("filter_dim", None)
filter_dim = self.data_store.get_default("filter_dim", None)
PlotSeparationOfScales(self.test_data, plot_folder=self.plot_path, time_dim=time_dim,
window_dim=window_dim, target_dim=target_dim, **{"filter_dim": filter_dim})
except Exception as e:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment