diff --git a/mlair/data_handler/abstract_data_handler.py b/mlair/data_handler/abstract_data_handler.py
index 7ef5b1644dbec037bb54016486be1fe5b515f6d9..9ea163fcad2890580e9c44e4bda0627d6419dc9f 100644
--- a/mlair/data_handler/abstract_data_handler.py
+++ b/mlair/data_handler/abstract_data_handler.py
@@ -32,8 +32,19 @@ class AbstractDataHandler(object):
def own_args(cls, *args):
"""Return all arguments (including kwonlyargs)."""
arg_spec = inspect.getfullargspec(cls)
- list_of_args = arg_spec.args + arg_spec.kwonlyargs
- return remove_items(list_of_args, list(args))
+ list_of_args = arg_spec.args + arg_spec.kwonlyargs + cls.super_args()
+ return list(set(remove_items(list_of_args, list(args))))
+
+ @classmethod
+ def super_args(cls):
+ args = []
+ for super_cls in cls.__mro__:
+ if super_cls == cls:
+ continue
+ if hasattr(super_cls, "own_args"):
+ # args.extend(super_cls.own_args())
+ args.extend(getattr(super_cls, "own_args")())
+ return list(set(args))
@classmethod
def store_attributes(cls) -> list:
diff --git a/mlair/data_handler/data_handler_mixed_sampling.py b/mlair/data_handler/data_handler_mixed_sampling.py
index 79e40424549a7c2e16c08f618e649f08984919fa..c6b612a72f03f231280392c66a4472549d0ef030 100644
--- a/mlair/data_handler/data_handler_mixed_sampling.py
+++ b/mlair/data_handler/data_handler_mixed_sampling.py
@@ -183,7 +183,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
return max(self.filter_order)
def apply_filter(self):
- DataHandlerFirFilterSingleStation.apply_filter()
+ DataHandlerFirFilterSingleStation.apply_filter(self)
def create_filter_index(self, add_unfiltered_index=True) -> pd.Index:
return DataHandlerFirFilterSingleStation.create_filter_index(self, add_unfiltered_index=add_unfiltered_index)
@@ -214,8 +214,8 @@ class DataHandlerMixedSamplingWithFirFilter(DataHandlerFirFilter):
_requirements = list(set(data_handler.requirements() + DataHandlerFirFilter.requirements()))
-class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixedSamplingWithFirFilterSingleStation,
- DataHandlerClimateFirFilterSingleStation):
+class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerClimateFirFilterSingleStation,
+ DataHandlerMixedSamplingWithFirFilterSingleStation):
_requirements1 = DataHandlerClimateFirFilterSingleStation.requirements()
_requirements2 = DataHandlerMixedSamplingWithFirFilterSingleStation.requirements()
_requirements = list(set(_requirements1 + _requirements2))
diff --git a/mlair/data_handler/data_handler_with_filter.py b/mlair/data_handler/data_handler_with_filter.py
index 93118bafa77004fbabf2020309bcad99c3c86e34..1925015a6170e863e9ddf057b46aec665cabd3fc 100644
--- a/mlair/data_handler/data_handler_with_filter.py
+++ b/mlair/data_handler/data_handler_with_filter.py
@@ -329,11 +329,12 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
"""
_requirements = DataHandlerFirFilterSingleStation.requirements()
- _hash = DataHandlerFirFilterSingleStation._hash + ["apriori_type", "apriori_sel_opts", "apriori_diurnal"]
+ _hash = DataHandlerFirFilterSingleStation._hash + ["apriori_type", "apriori_sel_opts", "apriori_diurnal",
+ "extend_length_opts"]
_store_attributes = DataHandlerFirFilterSingleStation.store_attributes() + ["apriori"]
def __init__(self, *args, apriori=None, apriori_type=None, apriori_diurnal=False, apriori_sel_opts=None,
- plot_path=None, name_affix=None, **kwargs):
+ plot_path=None, name_affix=None, extend_length_opts=None, **kwargs):
self.apriori_type = apriori_type
self.climate_filter_coeff = None # coefficents of the used FIR filter
self.apriori = apriori # exogenous apriori information or None to calculate from data (endogenous)
@@ -342,6 +343,7 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
self.apriori_sel_opts = apriori_sel_opts # ensure to separate exogenous and endogenous information
self.plot_path = plot_path # use this path to create insight plots
self.plot_name_affix = name_affix
+ self.extend_length_opts = extend_length_opts if extend_length_opts is not None else {}
super().__init__(*args, **kwargs)
@TimeTrackingWrapper
@@ -357,7 +359,7 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
apriori_diurnal=self.apriori_diurnal, sel_opts=self.apriori_sel_opts,
plot_path=self.plot_path, plot_name=plot_name,
minimum_length=self.window_history_size, new_dim=self.window_dim,
- station_name=self.station)
+ station_name=self.station, extend_length_opts=self.extend_length_opts)
self.climate_filter_coeff = climate_filter.filter_coefficients
# store apriori information: store all if residuum_stat method was used, otherwise just store initial apriori
@@ -367,8 +369,18 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
self.apriori = climate_filter.initial_apriori_data
self.all_apriori = climate_filter.apriori_data
- climate_filter_data = [c.sel({self.window_dim: slice(-self.window_history_size, 0)}) for c in
- climate_filter.filtered_data]
+ if isinstance(self.extend_length_opts, int):
+ climate_filter_data = [c.sel({self.window_dim: slice(-self.window_history_size, self.extend_length_opts)})
+ for c in climate_filter.filtered_data]
+ else:
+ climate_filter_data = []
+ for c in climate_filter.filtered_data:
+ coll_tmp = []
+ for v in c.coords[self.target_dim].values:
+ upper_lim = self.extend_length_opts.get(v, 0)
+ coll_tmp.append(c.sel({self.target_dim: v,
+ self.window_dim: slice(-self.window_history_size, upper_lim)}))
+ climate_filter_data.append(xr.concat(coll_tmp, self.target_dim))
# create input data with filter index
input_data = xr.concat(climate_filter_data, pd.Index(self.create_filter_index(add_unfiltered_index=False),
diff --git a/mlair/helpers/filter.py b/mlair/helpers/filter.py
index 3ac086791c7fd3f2580a7135ff5c29b9350f847f..3333382bce6e2800f5c9b15b8cd996994ac3fbaa 100644
--- a/mlair/helpers/filter.py
+++ b/mlair/helpers/filter.py
@@ -127,7 +127,7 @@ class ClimateFIRFilter(FIRFilter):
def __init__(self, data, fs, order, cutoff, window, time_dim, var_dim, apriori=None, apriori_type=None,
apriori_diurnal=False, sel_opts=None, plot_path=None, plot_name=None,
- minimum_length=None, new_dim=None, station_name=None):
+ minimum_length=None, new_dim=None, station_name=None, extend_length_opts: Union[dict, int] = 0):
"""
:param data: data to filter
:param fs: sampling frequency in 1/days -> 1d: fs=1 -> 1H: fs=24
@@ -143,6 +143,9 @@ class ClimateFIRFilter(FIRFilter):
residua is used ("residuum_stats").
:param apriori_diurnal: Use diurnal cycle as additional apriori information (only applicable for hourly
resoluted data). The mean anomaly of each hour is added to the apriori_type information.
+ :param extend_length_opts: shift information switch between historical data and apriori estimation by the given
+ values (default None). Must either be a dictionary with keys available in var_dim or a single value that is
+ applied to all data.
"""
#todo add extend_length_opts
# adjust all parts of code marked as todos
@@ -158,6 +161,7 @@ class ClimateFIRFilter(FIRFilter):
self.plot_path = plot_path
self.plot_name = plot_name # ToDo: is there a difference between station_name and plot_name???
self.plot_data = []
+ self.extend_length_opts = extend_length_opts
super().__init__(data, fs, order, cutoff, window, var_dim, time_dim, station_name)
def run(self):
@@ -189,20 +193,20 @@ class ClimateFIRFilter(FIRFilter):
for i in range(len(self.order)):
logging.info(f"{self.plot_name}: start filter for order {self.order[i]}")
# calculate climatological filter
- # ToDo: remove all methods except the vectorized version
_minimum_length = self._minimum_length(self.order, self.minimum_length, i, self.window)
fi, hi, apriori, plot_data = self.clim_filter(input_data, self.fs, self.cutoff[i], self.order[i],
apriori=apriori_list[i], sel_opts=self.sel_opts,
sampling=sampling, time_dim=self.time_dim,
window=self.window, var_dim=self.var_dim,
minimum_length=_minimum_length, new_dim=new_dim,
- plot_dates=plot_dates, station_name=self.station_name) #todo add extend_length_opts here
+ plot_dates=plot_dates, station_name=self.station_name,
+ extend_length_opts=self.extend_length_opts)
logging.info(f"{self.plot_name}: finished clim_filter calculation")
if self.minimum_length is None:
filtered.append(fi)
else:
- filtered.append(fi.sel({new_dim: slice(-self.minimum_length, 0)})) # todo adjust to extend_length_opts (how does it work with different lengths)
+ filtered.append(fi.sel({new_dim: slice(-self.minimum_length, None)}))
h.append(hi)
gc.collect()
self.plot_data.append(plot_data)
@@ -211,7 +215,7 @@ class ClimateFIRFilter(FIRFilter):
# calculate residuum
logging.info(f"{self.plot_name}: calculate residuum")
coord_range = range(fi.coords[new_dim].values.min(), fi.coords[new_dim].values.max() + 1)
- if new_dim in input_data.coords: #todo does it work for adding nans and values (should result in nans)
+ if new_dim in input_data.coords:
input_data = input_data.sel({new_dim: coord_range}) - fi
else:
input_data = self._shift_data(input_data, coord_range, self.time_dim, new_dim) - fi
@@ -239,7 +243,7 @@ class ClimateFIRFilter(FIRFilter):
if self.minimum_length is None:
filtered.append(input_data)
else:
- filtered.append(input_data.sel({new_dim: slice(-self.minimum_length, 0)})) #todo include extend_length_opts (how about different lengths again
+ filtered.append(input_data.sel({new_dim: slice(-self.minimum_length, None)}))
self._filtered = filtered
self._h = h
@@ -248,7 +252,7 @@ class ClimateFIRFilter(FIRFilter):
# visualize
if self.plot_path is not None:
try:
- self.PlotClimateFirFilter(self.plot_path, self.plot_data, sampling, plot_name)
+ self.PlotClimateFirFilter(self.plot_path, self.plot_data, sampling, self.plot_name) # not working when t0 != 0
except Exception as e:
logging.info(f"Could not plot climate fir filter due to following reason:\n{e}")
@@ -497,20 +501,22 @@ class ClimateFIRFilter(FIRFilter):
:returns: combined data array
"""
# check if shift indicated by extend_length_seperator is inside the outer interval limits
- assert (extend_length_separator > -extend_length_history) and (extend_length_separator < extend_length_future)
+ # assert (extend_length_separator > -extend_length_history) and (extend_length_separator < extend_length_future)
# prepare historical data / observation
if new_dim not in data.coords:
- history = self._shift_data(data, range(int(-extend_length_history), extend_length_seperator + 1),
+ history = self._shift_data(data, range(int(-extend_length_history), extend_length_separator + 1),
time_dim, new_dim)
else:
- history = data.sel({new_dim: slice(int(-extend_length_history), extend_length_seperator)})
+ history = data.sel({new_dim: slice(int(-extend_length_history), extend_length_separator)})
# prepare climatological statistics
if new_dim not in apriori.coords:
- future = self._shift_data(apriori, range(extend_length_seperator + 1, extend_length_future),
+ future = self._shift_data(apriori, range(extend_length_separator + 1,
+ extend_length_separator + extend_length_future),
time_dim, new_dim)
else:
- future = apriori.sel({new_dim: slice(extend_length_seperator + 1, extend_length_future)})
+ future = apriori.sel({new_dim: slice(extend_length_separator + 1,
+ extend_length_separator + extend_length_future)})
# combine historical data [t0-length,t0+sep] and climatological statistics [t0+sep+1,t0+length]
filter_input_data = xr.concat([history.dropna(time_dim), future], dim=new_dim, join="left")
@@ -624,10 +630,10 @@ class ClimateFIRFilter(FIRFilter):
def clim_filter(self, data, fs, cutoff_high, order, apriori=None, sel_opts=None,
sampling="1d", time_dim="datetime", var_dim="variables", window: Union[str, Tuple] = "hamming",
minimum_length=None, new_dim="window", plot_dates=None, station_name=None,
- extend_length_opts: dict = None):
+ extend_length_opts: Union[dict, int] = None):
logging.debug(f"{station_name}: extend apriori")
- extend_length_opts = extend_length_opts or {}
+ extend_opts = extend_length_opts if extend_length_opts is not None else {}
# calculate apriori information from data if not given and extend its range if not sufficient long enough
if apriori is None:
@@ -656,13 +662,14 @@ class ClimateFIRFilter(FIRFilter):
logging.info(f"{station_name} ({var}): sel data")
_start, _end = self._get_year_interval(data, time_dim)
- extend_length_opts_var = extend_length_opts.get(var, 0)
+ extend_opts_var = extend_opts.get(var, 0) if isinstance(extend_opts, dict) else extend_opts
filt_coll = []
for _year in range(_start, _end + 1):
logging.debug(f"{station_name} ({var}): year={_year}")
# select observations and apriori data
- time_slice = self._create_time_range_extend(_year, sampling, extend_length_history)
+ time_slice = self._create_time_range_extend(
+ _year, sampling, max(extend_length_history, extend_length_future + extend_opts_var))
d = data.sel({var_dim: [var], time_dim: time_slice})
a = apriori.sel({var_dim: [var], time_dim: time_slice})
if len(d.coords[time_dim]) == 0: # no data at all for this year
@@ -671,7 +678,7 @@ class ClimateFIRFilter(FIRFilter):
# combine historical data / observation [t0-length,t0] and climatological statistics [t0+1,t0+length]
filter_input_data = self.combine_observation_and_apriori(d, a, time_dim, new_dim, extend_length_history,
extend_length_future,
- extend_length_separator=extend_length_opts_var)
+ extend_length_separator=extend_opts_var)
# select only data for current year
try:
@@ -690,7 +697,7 @@ class ClimateFIRFilter(FIRFilter):
# trim data if required
trimmed = self._trim_data_to_minimum_length(filt, extend_length_history, new_dim, minimum_length,
- extend_length_opts=extend_length_opts_var)
+ extend_length_opts=extend_opts_var)
filt_coll.append(trimmed)
# visualization
@@ -707,7 +714,7 @@ class ClimateFIRFilter(FIRFilter):
res = xr.concat(coll, var_dim) #todo does this works with different extend_length_opts (is data trimmed or filled with nans, 2nd is target)
# create result array with same shape like input data, gaps are filled by nans
- res_full = self._create_full_filter_result_array(data, res, new_dim, station_name) #todo does it still works with extend_length_opts
+ res_full = self._create_full_filter_result_array(data, res, new_dim, station_name)
return res_full, h, apriori, plot_data
@staticmethod
diff --git a/mlair/helpers/time_tracking.py b/mlair/helpers/time_tracking.py
index cf366db88adc524e90c2b771bef77c71ee5a9502..5df695b9eee5352152c3189111bacf2fe05a2cb3 100644
--- a/mlair/helpers/time_tracking.py
+++ b/mlair/helpers/time_tracking.py
@@ -41,7 +41,10 @@ class TimeTrackingWrapper:
def __get__(self, instance, cls):
"""Create bound method object and supply self argument to the decorated method."""
- return types.MethodType(self, instance)
+ if instance is None:
+ return self
+ else:
+ return types.MethodType(self, instance)
class TimeTracking(object):
diff --git a/mlair/plotting/abstract_plot_class.py b/mlair/plotting/abstract_plot_class.py
index c91dbec78c4bc990cc9c40c3afb6c506b62928d8..7a91c2269ccd03608bcdbe67a634156f55fde91f 100644
--- a/mlair/plotting/abstract_plot_class.py
+++ b/mlair/plotting/abstract_plot_class.py
@@ -59,7 +59,7 @@ class AbstractPlotClass:
if not os.path.exists(plot_folder):
os.makedirs(plot_folder)
self.plot_folder = plot_folder
- self.plot_name = plot_name.replace("/", "_")
+ self.plot_name = plot_name.replace("/", "_") if plot_name is not None else plot_name
self.resolution = resolution
if rc_params is None:
rc_params = {'axes.labelsize': 'large',
@@ -71,6 +71,9 @@ class AbstractPlotClass:
self.rc_params = rc_params
self._update_rc_params()
+ def __del__(self):
+ plt.close('all')
+
def _plot(self, *args):
"""Abstract plot class needs to be implemented in inheritance."""
raise NotImplementedError