Skip to content
Snippets Groups Projects
Commit 43f7d659 authored by leufen1's avatar leufen1
Browse files

use class variables for chem and meteo names, is now able to distinguish for... 

use class variables for chem and meteo names, is now able to distinguish for all parameters of the data handler if provided as dict with respective keys, only use test stations for block mse calculation
parent aa858458
No related branches found
No related tags found
4 merge requests!430update recent developments,!413update release branch,!412Resolve "release v2.0.0",!378Resolve "filter dh: differentiate between history offset and information offset"
Pipeline #89105 passed
Stage: test
Stage: docs
Stage: pages
Stage: deploy
Show whitespace changes
Inline Side-by-side
mlair/data_handler/data_handler_mixed_sampling.py
+ 30
− 18
View file @ 43f7d659
......@@ -244,11 +244,11 @@ class DataHandlerMixedSamplingWithClimateFirFilter(DataHandlerClimateFirFilter):
def build(cls, station: str, **kwargs):
sp_keys = {k: copy.deepcopy(kwargs[k]) for k in cls.data_handler.requirements() if k in kwargs}
filter_add_unfiltered = kwargs.get("filter_add_unfiltered", False)
sp_keys = cls.build_update_kwargs(sp_keys, dh_type="filtered")
sp_keys = cls.build_update_transformation(sp_keys, dh_type="filtered")
sp = cls.data_handler(station, **sp_keys)
if filter_add_unfiltered is True:
sp_keys = {k: copy.deepcopy(kwargs[k]) for k in cls.data_handler_unfiltered.requirements() if k in kwargs}
sp_keys = cls.build_update_kwargs(sp_keys, dh_type="unfiltered")
sp_keys = cls.build_update_transformation(sp_keys, dh_type="unfiltered")
sp_unfiltered = cls.data_handler_unfiltered(station, **sp_keys)
else:
sp_unfiltered = None
......@@ -256,7 +256,7 @@ class DataHandlerMixedSamplingWithClimateFirFilter(DataHandlerClimateFirFilter):
return cls(sp, data_handler_class_unfiltered=sp_unfiltered, **dp_args)
@classmethod
def build_update_kwargs(cls, kwargs_dict, dh_type="filtered"):
def build_update_transformation(cls, kwargs_dict, dh_type="filtered"):
if "transformation" in kwargs_dict:
trafo_opts = kwargs_dict.get("transformation")
if isinstance(trafo_opts, dict):
......@@ -313,6 +313,8 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
data_handler_unfiltered = DataHandlerMixedSamplingSingleStation
_requirements = list(set(data_handler_climate_fir.requirements() + data_handler_fir[0].requirements() +
data_handler_fir[1].requirements() + data_handler_unfiltered.requirements()))
chem_indicator = "chem"
meteo_indicator = "meteo"
def __init__(self, data_handler_class_chem, data_handler_class_meteo, data_handler_class_chem_unfiltered,
data_handler_class_meteo_unfiltered, chem_vars, meteo_vars, *args, **kwargs):
......@@ -351,32 +353,32 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
if len(chem_vars) > 0:
sp_keys = {k: copy.deepcopy(kwargs[k]) for k in cls.data_handler_climate_fir.requirements() if k in kwargs}
sp_keys = cls.build_update_kwargs(sp_keys, dh_type="filtered_chem")
sp_keys = cls.build_update_transformation(sp_keys, dh_type="filtered_chem")
cls.prepare_build(sp_keys, chem_vars, "chem")
cls.prepare_build(sp_keys, chem_vars, cls.chem_indicator)
sp_chem = cls.data_handler_climate_fir(station, **sp_keys)
if filter_add_unfiltered is True:
sp_keys = {k: copy.deepcopy(kwargs[k]) for k in cls.data_handler_unfiltered.requirements() if k in kwargs}
sp_keys = cls.build_update_kwargs(sp_keys, dh_type="unfiltered_chem")
cls.prepare_build(sp_keys, chem_vars, "chem")
sp_keys = cls.build_update_transformation(sp_keys, dh_type="unfiltered_chem")
cls.prepare_build(sp_keys, chem_vars, cls.chem_indicator)
sp_chem_unfiltered = cls.data_handler_unfiltered(station, **sp_keys)
if len(meteo_vars) > 0:
if cls.data_handler_fir_pos is None:
if "extend_length_opts" in kwargs:
if isinstance(kwargs["extend_length_opts"], dict) and "meteo" not in kwargs["extend_length_opts"].keys():
if isinstance(kwargs["extend_length_opts"], dict) and cls.meteo_indicator not in kwargs["extend_length_opts"].keys():
cls.data_handler_fir_pos = 0 # use faster fir version without climate estimate
else:
cls.data_handler_fir_pos = 1 # use slower fir version with climate estimate
else:
cls.data_handler_fir_pos = 0 # use faster fir version without climate estimate
sp_keys = {k: copy.deepcopy(kwargs[k]) for k in cls.data_handler_fir[cls.data_handler_fir_pos].requirements() if k in kwargs}
sp_keys = cls.build_update_kwargs(sp_keys, dh_type="filtered_meteo")
cls.prepare_build(sp_keys, meteo_vars, "meteo")
sp_keys = cls.build_update_transformation(sp_keys, dh_type="filtered_meteo")
cls.prepare_build(sp_keys, meteo_vars, cls.meteo_indicator)
sp_meteo = cls.data_handler_fir[cls.data_handler_fir_pos](station, **sp_keys)
if filter_add_unfiltered is True:
sp_keys = {k: copy.deepcopy(kwargs[k]) for k in cls.data_handler_unfiltered.requirements() if k in kwargs}
sp_keys = cls.build_update_kwargs(sp_keys, dh_type="unfiltered_meteo")
cls.prepare_build(sp_keys, meteo_vars, "meteo")
sp_keys = cls.build_update_transformation(sp_keys, dh_type="unfiltered_meteo")
cls.prepare_build(sp_keys, meteo_vars, cls.meteo_indicator)
sp_meteo_unfiltered = cls.data_handler_unfiltered(station, **sp_keys)
dp_args = {k: copy.deepcopy(kwargs[k]) for k in cls.own_args("id_class") if k in kwargs}
......@@ -385,10 +387,20 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
@classmethod
def prepare_build(cls, kwargs, var_list, var_type):
kwargs.update({"variables": var_list})
cls.adjust_window_opts(var_type, "window_history_size", kwargs)
cls.adjust_window_opts(var_type, "window_history_offset", kwargs)
cls.adjust_window_opts(var_type, "window_history_end", kwargs)
cls.adjust_window_opts(var_type, "extend_length_opts", kwargs)
for k in list(kwargs.keys()):
v = kwargs[k]
if isinstance(v, dict):
if len(set(v.keys()).intersection({cls.chem_indicator, cls.meteo_indicator})) > 0:
try:
new_v = kwargs.pop(k)
kwargs[k] = new_v[var_type]
except KeyError:
pass
#
# cls.adjust_window_opts(var_type, "window_history_size", kwargs)
# cls.adjust_window_opts(var_type, "window_history_offset", kwargs)
# cls.adjust_window_opts(var_type, "window_history_end", kwargs)
# cls.adjust_window_opts(var_type, "extend_length_opts", kwargs)
@staticmethod
def adjust_window_opts(key: str, parameter_name: str, kwargs: dict):
......@@ -420,7 +432,7 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
# chem transformation
if len(chem_vars) > 0:
kwargs_chem = copy.deepcopy(kwargs)
cls.prepare_build(kwargs_chem, chem_vars, "chem")
cls.prepare_build(kwargs_chem, chem_vars, cls.chem_indicator)
dh_transformation = (cls.data_handler_climate_fir, cls.data_handler_unfiltered)
transformation_chem = super().transformation(set_stations, tmp_path=tmp_path,
dh_transformation=dh_transformation, **kwargs_chem)
......@@ -428,7 +440,7 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
# meteo transformation
if len(meteo_vars) > 0:
kwargs_meteo = copy.deepcopy(kwargs)
cls.prepare_build(kwargs_meteo, meteo_vars, "meteo")
cls.prepare_build(kwargs_meteo, meteo_vars, cls.meteo_indicator)
dh_transformation = (cls.data_handler_fir[cls.data_handler_fir_pos or 0], cls.data_handler_unfiltered)
transformation_meteo = super().transformation(set_stations, tmp_path=tmp_path,
dh_transformation=dh_transformation, **kwargs_meteo)
......
mlair/run_modules/post_processing.py
+ 1
− 1
View file @ 43f7d659
......@@ -184,7 +184,7 @@ class PostProcessing(RunEnvironment):
against the number of observations and diversity ot stations.
"""
path = self.data_store.get("forecast_path")
all_stations = self.data_store.get("stations")
all_stations = self.data_store.get("stations", "test")
start = self.data_store.get("start", "test")
end = self.data_store.get("end", "test")
index_dim = self.index_dim
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment