From 19acec962d01e17ebcaa14b571200b8aafe9ec87 Mon Sep 17 00:00:00 2001
From: leufen1 <l.leufen@fz-juelich.de>
Date: Fri, 14 Jan 2022 11:22:43 +0100
Subject: [PATCH] added some doc strings, removed outdated lines, updated some
tests
---
.../data_handler/data_handler_with_filter.py | 34 +++--
mlair/helpers/filter.py | 117 ++++++++----------
mlair/plotting/data_insight_plotting.py | 30 ++---
test/test_helpers/test_filter.py | 23 ++--
4 files changed, 93 insertions(+), 111 deletions(-)
diff --git a/mlair/data_handler/data_handler_with_filter.py b/mlair/data_handler/data_handler_with_filter.py
index 448db592..761c7c8d 100644
--- a/mlair/data_handler/data_handler_with_filter.py
+++ b/mlair/data_handler/data_handler_with_filter.py
@@ -190,7 +190,7 @@ class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
def apply_filter(self):
"""Apply FIR filter only on inputs."""
fir = FIRFilter(self.input_data.astype("float32"), self.fs, self.filter_order, self.filter_cutoff_freq,
- self.filter_window_type, self.target_dim, self.time_dim, station_name=self.station[0],
+ self.filter_window_type, self.target_dim, self.time_dim, display_name=self.station[0],
minimum_length=self.window_history_size, offset=self.window_history_offset, plot_path=self.plot_path)
self.fir_coeff = fir.filter_coefficients
filter_data = fir.filtered_data
@@ -326,12 +326,15 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
apriori_type is None or `zeros`, and a climatology of the residuum is used for `residuum_stats`.
:param apriori_diurnal: use diurnal anomalies of each hour as addition to the apriori information type chosen by
parameter apriori_type. This is only applicable for hourly resolution data.
+ :param apriori_sel_opts: specify some parameters to select a subset of data before calculating the apriori
+ information. Use this parameter for example, if apriori shall only calculated on a shorter time period than
+ available in given data.
:param extend_length_opts: use this parameter to use future data in the filter calculation. This parameter does not
- affect the size of the history samples as this is handled by the window_history_offset parameter. Example: set
+ affect the size of the history samples as this is handled by the window_history_size parameter. Example: set
extend_length_opts=7*24 to use the observation of the next 7 days to calculate the filtered components. Which
- data are finally used for the input samples is not affected by these 7 days. In case the window_history_offset
- parameter exceeds the `extend_length_opts`, it has also an influence on the filter component calculation as it
- will return also climatological estimates.
+ data are finally used for the input samples is not affected by these 7 days. In case the range of history sample
+ exceeds the horizon of extend_length_opts, the history sample will also include data from climatological
+ estimates.
"""
DEFAULT_EXTEND_LENGTH_OPTS = 0
_hash = DataHandlerFirFilterSingleStation._hash + ["apriori_type", "apriori_sel_opts", "apriori_diurnal",
@@ -339,14 +342,13 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
_store_attributes = DataHandlerFirFilterSingleStation.store_attributes() + ["apriori"]
def __init__(self, *args, apriori=None, apriori_type=None, apriori_diurnal=False, apriori_sel_opts=None,
- name_affix=None, extend_length_opts=DEFAULT_EXTEND_LENGTH_OPTS, **kwargs):
+ extend_length_opts=DEFAULT_EXTEND_LENGTH_OPTS, **kwargs):
self.apriori_type = apriori_type
self.climate_filter_coeff = None # coefficents of the used FIR filter
self.apriori = apriori # exogenous apriori information or None to calculate from data (endogenous)
self.apriori_diurnal = apriori_diurnal
self.all_apriori = None # collection of all apriori information
self.apriori_sel_opts = apriori_sel_opts # ensure to separate exogenous and endogenous information
- self.plot_name_affix = name_affix
self.extend_length_opts = extend_length_opts
super().__init__(*args, **kwargs)
@@ -362,7 +364,7 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
apriori_diurnal=self.apriori_diurnal, sel_opts=self.apriori_sel_opts,
plot_path=self.plot_path,
minimum_length=self.window_history_size, new_dim=self.window_dim,
- station_name=self.station[0], extend_length_opts=self.extend_length_opts,
+ display_name=self.station[0], extend_length_opts=self.extend_length_opts,
offset=self.window_history_end)
self.climate_filter_coeff = climate_filter.filter_coefficients
@@ -431,21 +433,15 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
def make_history_window(self, dim_name_of_inputs: str, window: int, dim_name_of_shift: str) -> None:
"""
- Create a xr.DataArray containing history data.
-
- Shift the data window+1 times and return a xarray which has a new dimension 'window' containing the shifted
- data. This is used to represent history in the data. Results are stored in history attribute.
+ Create a xr.DataArray containing history data. As 'input_data' already consists of a dimension 'window', this
+ method only shifts the data along 'window' dimension x times where x is given by 'window_history_offset'.
+ Results are stored in history attribute.
:param dim_name_of_inputs: Name of dimension which contains the input variables
- :param window: number of time steps to look back in history
- Note: window will be treated as negative value. This should be in agreement with looking back on
- a time line. Nonetheless positive values are allowed but they are converted to its negative
- expression
+ :param window: this parameter is not used in the inherited method
:param dim_name_of_shift: Dimension along shift will be applied
"""
- # self.history = self.input_data.__deepcopy__()
- #todo stopped here. maybe there is nothing to do anymore?
- data = self.input_data #todo trim data to be only in range slice(None, self.window_history_offset) ??
+ data = self.input_data
sampling = {"daily": "D", "hourly": "h"}.get(to_list(self.sampling)[0])
data.coords[dim_name_of_shift] = data.coords[dim_name_of_shift] - np.timedelta64(self.window_history_offset,
sampling)
diff --git a/mlair/helpers/filter.py b/mlair/helpers/filter.py
index 3e93ebc6..03065402 100644
--- a/mlair/helpers/filter.py
+++ b/mlair/helpers/filter.py
@@ -2,8 +2,6 @@ import gc
import warnings
from typing import Union, Callable, Tuple, Dict, Any
import logging
-import os
-import time
import datetime
import numpy as np
@@ -19,7 +17,7 @@ from mlair.helpers import to_list, TimeTrackingWrapper, TimeTracking
class FIRFilter:
from mlair.plotting.data_insight_plotting import PlotFirFilter
- def __init__(self, data, fs, order, cutoff, window, var_dim, time_dim, station_name=None, minimum_length=None, offset=0, plot_path=None):
+ def __init__(self, data, fs, order, cutoff, window, var_dim, time_dim, display_name=None, minimum_length=None, offset=0, plot_path=None):
self._filtered = []
self._h = []
self.data = data
@@ -29,14 +27,14 @@ class FIRFilter:
self.window = window
self.var_dim = var_dim
self.time_dim = time_dim
- self.station_name = station_name
+ self.display_name = display_name
self.minimum_length = minimum_length
self.offset = offset
self.plot_path = plot_path
self.run()
def run(self):
- logging.info(f"{self.station_name}: start {self.__class__.__name__}")
+ logging.info(f"{self.display_name}: start {self.__class__.__name__}")
filtered = []
h = []
input_data = self.data.__deepcopy__()
@@ -50,7 +48,7 @@ class FIRFilter:
for i in range(len(self.order)):
# apply filter
fi, hi = self.fir_filter(input_data, self.fs, self.cutoff[i], self.order[i], time_dim=self.time_dim,
- var_dim=self.var_dim, window=self.window, station_name=self.station_name)
+ var_dim=self.var_dim, window=self.window, display_name=self.display_name)
filtered.append(fi)
h.append(hi)
@@ -69,7 +67,7 @@ class FIRFilter:
# visualize
if self.plot_path is not None:
try:
- self.PlotFirFilter(self.plot_path, plot_data, self.station_name) # not working when t0 != 0
+ self.PlotFirFilter(self.plot_path, plot_data, self.display_name) # not working when t0 != 0
except Exception as e:
logging.info(f"Could not plot climate fir filter due to following reason:\n{e}")
@@ -106,7 +104,7 @@ class FIRFilter:
@TimeTrackingWrapper
def fir_filter(self, data, fs, cutoff_high, order, sampling="1d", time_dim="datetime", var_dim="variables", window: Union[str, Tuple] = "hamming",
- minimum_length=None, new_dim="window", plot_dates=None, station_name=None):
+ minimum_length=None, new_dim="window", plot_dates=None, display_name=None):
# calculate FIR filter coefficients
h = self._calculate_filter_coefficients(window, order, cutoff_high, fs)
@@ -121,7 +119,7 @@ class FIRFilter:
filtered = xr.concat(coll, var_dim)
# create result array with same shape like input data, gaps are filled by nans
- filtered = self._create_full_filter_result_array(data, filtered, time_dim, station_name)
+ filtered = self._create_full_filter_result_array(data, filtered, time_dim, display_name)
return filtered, h
@staticmethod
@@ -145,7 +143,7 @@ class FIRFilter:
@staticmethod
def _create_full_filter_result_array(template_array: xr.DataArray, result_array: xr.DataArray, new_dim: str,
- station_name: str = None) -> xr.DataArray:
+ display_name: str = None) -> xr.DataArray:
"""
Create result filter array with same shape line given template data (should be the original input data before
filtering the data). All gaps are filled by nans.
@@ -153,9 +151,9 @@ class FIRFilter:
:param template_array: this array is used as template for shape and ordering of dims
:param result_array: array with data that are filled into template
:param new_dim: new dimension which is shifted/appended to/at the end (if present or not)
- :param station_name: string that is attached to logging (default None)
+ :param display_name: string that is attached to logging (default None)
"""
- logging.debug(f"{station_name}: create res_full")
+ logging.debug(f"{display_name}: create res_full")
new_coords = {**{k: template_array.coords[k].values for k in template_array.coords if k != new_dim},
new_dim: result_array.coords[new_dim]}
dims = [*template_array.dims, new_dim] if new_dim not in template_array.dims else template_array.dims
@@ -168,7 +166,7 @@ class ClimateFIRFilter(FIRFilter):
def __init__(self, data, fs, order, cutoff, window, time_dim, var_dim, apriori=None, apriori_type=None,
apriori_diurnal=False, sel_opts=None, plot_path=None,
- minimum_length=None, new_dim=None, station_name=None, extend_length_opts: int = 0,
+ minimum_length=None, new_dim=None, display_name=None, extend_length_opts: int = 0,
offset: Union[dict, int] = 0):
"""
:param data: data to filter
@@ -185,6 +183,9 @@ class ClimateFIRFilter(FIRFilter):
residua is used ("residuum_stats").
:param apriori_diurnal: Use diurnal cycle as additional apriori information (only applicable for hourly
resoluted data). The mean anomaly of each hour is added to the apriori_type information.
+ :param sel_opts: specify some parameters to select a subset of data before calculating the apriori information.
+ Use this parameter for example, if apriori shall only calculated on a shorter time period than available in
+ given data.
:param extend_length_opts: shift information switch between historical data and apriori estimation by the given
values (default None). Must either be a dictionary with keys available in var_dim or a single value that is
applied to all data. This parameter has only influence on which information is available at t0 for the
@@ -202,7 +203,7 @@ class ClimateFIRFilter(FIRFilter):
self.new_dim = new_dim
self.plot_data = []
self.extend_length_opts = extend_length_opts
- super().__init__(data, fs, order, cutoff, window, var_dim, time_dim, station_name=station_name,
+ super().__init__(data, fs, order, cutoff, window, var_dim, time_dim, display_name=display_name,
minimum_length=minimum_length, plot_path=plot_path, offset=offset)
def run(self):
@@ -211,17 +212,17 @@ class ClimateFIRFilter(FIRFilter):
if self.sel_opts is not None:
self.sel_opts = self.sel_opts if isinstance(self.sel_opts, dict) else {self.time_dim: self.sel_opts}
sampling = {1: "1d", 24: "1H"}.get(int(self.fs))
- logging.debug(f"{self.station_name}: create diurnal_anomalies")
+ logging.debug(f"{self.display_name}: create diurnal_anomalies")
if self.apriori_diurnal is True and sampling == "1H":
diurnal_anomalies = self.create_seasonal_hourly_mean(self.data, self.time_dim, sel_opts=self.sel_opts,
sampling=sampling, as_anomaly=True)
else:
diurnal_anomalies = 0
- logging.debug(f"{self.station_name}: create monthly apriori")
+ logging.debug(f"{self.display_name}: create monthly apriori")
if self._apriori is None:
self._apriori = self.create_monthly_mean(self.data, self.time_dim, sel_opts=self.sel_opts,
sampling=sampling) + diurnal_anomalies
- logging.debug(f"{self.station_name}: apriori shape = {self._apriori.shape}")
+ logging.debug(f"{self.display_name}: apriori shape = {self._apriori.shape}")
apriori_list = to_list(self._apriori)
input_data = self.data.__deepcopy__()
@@ -232,19 +233,19 @@ class ClimateFIRFilter(FIRFilter):
new_dim = self._create_tmp_dimension(input_data) if self.new_dim is None else self.new_dim
for i in range(len(self.order)):
- logging.info(f"{self.station_name}: start filter for order {self.order[i]}")
+ logging.info(f"{self.display_name}: start filter for order {self.order[i]}")
# calculate climatological filter
next_order = self._next_order(self.order, 0, i, self.window)
fi, input_data, hi, apriori, plot_data = self.clim_filter(input_data, self.fs, self.cutoff[i], self.order[i],
- apriori=apriori_list[i], sel_opts=self.sel_opts,
- sampling=sampling, time_dim=self.time_dim,
- window=self.window, var_dim=self.var_dim,
- minimum_length=self.minimum_length, new_dim=new_dim,
- plot_dates=plot_dates, station_name=self.station_name,
- extend_opts=self.extend_length_opts,
- offset=self.offset, next_order=next_order)
-
- logging.info(f"{self.station_name}: finished clim_filter calculation")
+ apriori=apriori_list[i], sel_opts=self.sel_opts,
+ sampling=sampling, time_dim=self.time_dim,
+ window=self.window, var_dim=self.var_dim,
+ minimum_length=self.minimum_length, new_dim=new_dim,
+ plot_dates=plot_dates, display_name=self.display_name,
+ extend_opts=self.extend_length_opts,
+ offset=self.offset, next_order=next_order)
+
+ logging.info(f"{self.display_name}: finished clim_filter calculation")
if self.minimum_length is None:
filtered.append(fi.sel({new_dim: slice(None, self.offset)}))
else:
@@ -255,7 +256,7 @@ class ClimateFIRFilter(FIRFilter):
plot_dates = {e["viz_date"] for e in plot_data}
# calculate residuum
- logging.info(f"{self.station_name}: calculate residuum")
+ logging.info(f"{self.display_name}: calculate residuum")
coord_range = range(fi.coords[new_dim].values.min(), fi.coords[new_dim].values.max() + 1)
if new_dim in input_data.coords:
input_data = input_data.sel({new_dim: coord_range}) - fi
@@ -264,14 +265,14 @@ class ClimateFIRFilter(FIRFilter):
# create new apriori information for next iteration if no further apriori is provided
if len(apriori_list) < len(self.order):
- logging.info(f"{self.station_name}: create diurnal_anomalies")
+ logging.info(f"{self.display_name}: create diurnal_anomalies")
if self.apriori_diurnal is True and sampling == "1H":
diurnal_anomalies = self.create_seasonal_hourly_mean(input_data.sel({new_dim: 0}, drop=True),
self.time_dim, sel_opts=self.sel_opts,
sampling=sampling, as_anomaly=True)
else:
diurnal_anomalies = 0
- logging.info(f"{self.station_name}: create monthly apriori")
+ logging.info(f"{self.display_name}: create monthly apriori")
if self.apriori_type is None or self.apriori_type == "zeros": # zero version
apriori_list.append(xr.zeros_like(apriori_list[i]) + diurnal_anomalies)
elif self.apriori_type == "residuum_stats": # calculate monthly statistic on residuum
@@ -295,7 +296,7 @@ class ClimateFIRFilter(FIRFilter):
# visualize
if self.plot_path is not None:
try:
- self.PlotClimateFirFilter(self.plot_path, self.plot_data, sampling, self.station_name)
+ self.PlotClimateFirFilter(self.plot_path, self.plot_data, sampling, self.display_name)
except Exception as e:
logging.info(f"Could not plot climate fir filter due to following reason:\n{e}")
@@ -460,7 +461,7 @@ class ClimateFIRFilter(FIRFilter):
@staticmethod
def extend_apriori(data: xr.DataArray, apriori: xr.DataArray, time_dim: str, sampling: str = "1d",
- station_name: str = None) -> xr.DataArray:
+ display_name: str = None) -> xr.DataArray:
"""
Extend time range of apriori information to span a longer period as data (or at least of equal length). This
method may not working properly if length of apriori contains data from less then one year.
@@ -471,7 +472,7 @@ class ClimateFIRFilter(FIRFilter):
apriori data.
:param time_dim: name of temporal dimension
:param sampling: sampling of data (e.g. "1m", "1d", default "1d")
- :param station_name: name to use for logging message (default None)
+ :param display_name: name to use for logging message (default None)
:returns: array which adjusted temporal coverage derived from apriori
"""
dates = data.coords[time_dim].values
@@ -479,7 +480,7 @@ class ClimateFIRFilter(FIRFilter):
# apriori starts after data
if dates[0] < apriori.coords[time_dim].values[0]:
- logging.debug(f"{station_name}: apriori starts after data")
+ logging.debug(f"{display_name}: apriori starts after data")
# add difference in full years
date_diff = abs(dates[0] - apriori.coords[time_dim].values[0]).astype("timedelta64[D]")
@@ -500,7 +501,7 @@ class ClimateFIRFilter(FIRFilter):
# apriori ends before data
if dates[-1] + np.timedelta64(365, "D") > apriori.coords[time_dim].values[-1]:
- logging.debug(f"{station_name}: apriori ends before data")
+ logging.debug(f"{display_name}: apriori ends before data")
# add difference in full years + 1 year (because apriori is used as future estimate)
date_diff = abs(dates[-1] - apriori.coords[time_dim].values[-1]).astype("timedelta64[D]")
@@ -572,42 +573,32 @@ class ClimateFIRFilter(FIRFilter):
plot_data = []
offset = 0 if offset is None else offset
extend_length_opts = 0 if extend_length_opts is None else extend_length_opts
- for viz_date in set(plot_dates).intersection(filtered.coords[time_dim].values):
+ for t0 in set(plot_dates).intersection(filtered.coords[time_dim].values):
try:
td_type = {"1d": "D", "1H": "h"}.get(sampling)
- # t0 = viz_date + np.timedelta64(int(offset), td_type)
- t0 = viz_date # offset should be irrelevant for visualization
- t_minus = viz_date + np.timedelta64(int(-extend_length_history), td_type)
+ t_minus = t0 + np.timedelta64(int(-extend_length_history), td_type)
t_plus = t0 + np.timedelta64(int(extend_length_future + 1), td_type)
- # t_plus = t0 + np.timedelta64(int(extend_length_future + extend_length_opts), td_type)
if new_dim not in data.coords:
- # tmp_filter_data = self._shift_data(data.sel({time_dim: slice(t_minus, t_plus)}),
- # range(int(-extend_length_history),
- # int(extend_length_future + extend_length_opts)),
- # time_dim,
- # new_dim).sel({time_dim: viz_date})
tmp_filter_data = self._shift_data(data.sel({time_dim: slice(t_minus, t_plus)}),
range(int(-extend_length_history),
int(extend_length_future + 1)),
time_dim,
- new_dim).sel({time_dim: viz_date})
+ new_dim).sel({time_dim: t0})
else:
tmp_filter_data = None
valid_start = int(filtered.window.min()) + int((len(h) + 1) / 2)
valid_end = min(extend_length_opts + offset + 1, int(filtered.window.max()) - int((len(h) + 1) / 2))
valid_range = range(valid_start, valid_end)
plot_data.append({"t0": t0,
- "viz_date": viz_date,
"var": variable_name,
- "filter_input": filter_input_data.sel({time_dim: viz_date}),
+ "filter_input": filter_input_data.sel({time_dim: t0}),
"filter_input_nc": tmp_filter_data,
"valid_range": valid_range,
"time_range": data.sel(
{time_dim: slice(t_minus, t_plus - np.timedelta64(1, td_type))}).coords[
time_dim].values,
"h": h,
- "new_dim": new_dim,
- "offset": offset})
+ "new_dim": new_dim})
except:
pass
return plot_data
@@ -667,7 +658,7 @@ class ClimateFIRFilter(FIRFilter):
@staticmethod
def _create_full_filter_result_array(template_array: xr.DataArray, result_array: xr.DataArray, new_dim: str,
- station_name: str = None) -> xr.DataArray:
+ display_name: str = None) -> xr.DataArray:
"""
Create result filter array with same shape line given template data (should be the original input data before
filtering the data). All gaps are filled by nans.
@@ -675,9 +666,9 @@ class ClimateFIRFilter(FIRFilter):
:param template_array: this array is used as template for shape and ordering of dims
:param result_array: array with data that are filled into template
:param new_dim: new dimension which is shifted/appended to/at the end (if present or not)
- :param station_name: string that is attached to logging (default None)
+ :param display_name: string that is attached to logging (default None)
"""
- logging.debug(f"{station_name}: create res_full")
+ logging.debug(f"{display_name}: create res_full")
new_coords = {**{k: template_array.coords[k].values for k in template_array.coords if k != new_dim},
new_dim: result_array.coords[new_dim]}
dims = [*template_array.dims, new_dim] if new_dim not in template_array.dims else template_array.dims
@@ -687,16 +678,16 @@ class ClimateFIRFilter(FIRFilter):
@TimeTrackingWrapper
def clim_filter(self, data, fs, cutoff_high, order, apriori=None, sel_opts=None,
sampling="1d", time_dim="datetime", var_dim="variables", window: Union[str, Tuple] = "hamming",
- minimum_length=0, next_order=0, new_dim="window", plot_dates=None, station_name=None,
+ minimum_length=0, next_order=0, new_dim="window", plot_dates=None, display_name=None,
extend_opts: int = 0, offset: int = 0):
- logging.debug(f"{station_name}: extend apriori")
+ logging.debug(f"{display_name}: extend apriori")
# calculate apriori information from data if not given and extend its range if not sufficient long enough
if apriori is None:
apriori = self.create_monthly_mean(data, time_dim, sel_opts=sel_opts, sampling=sampling)
apriori = apriori.astype(data.dtype)
- apriori = self.extend_apriori(data, apriori, time_dim, sampling, station_name=station_name)
+ apriori = self.extend_apriori(data, apriori, time_dim, sampling, display_name=display_name)
# calculate FIR filter coefficients
h = self._calculate_filter_coefficients(window, order, cutoff_high, fs)
@@ -717,13 +708,13 @@ class ClimateFIRFilter(FIRFilter):
coll_input = []
for var in reversed(data.coords[var_dim].values):
- logging.info(f"{station_name} ({var}): sel data")
+ logging.info(f"{display_name} ({var}): sel data")
_start, _end = self._get_year_interval(data, time_dim)
filt_coll = []
filt_input_coll = []
for _year in range(_start, _end + 1):
- logging.debug(f"{station_name} ({var}): year={_year}")
+ logging.debug(f"{display_name} ({var}): year={_year}")
# select observations and apriori data
time_slice = self._create_time_range_extend(
@@ -746,8 +737,8 @@ class ClimateFIRFilter(FIRFilter):
continue
# apply filter
- logging.debug(f"{station_name} ({var}): start filter convolve")
- with TimeTracking(name=f"{station_name} ({var}): filter convolve", logging_level=logging.DEBUG):
+ logging.debug(f"{display_name} ({var}): start filter convolve")
+ with TimeTracking(name=f"{display_name} ({var}): filter convolve", logging_level=logging.DEBUG):
filt = xr.apply_ufunc(fir_filter_convolve, filter_input_data,
input_core_dims=[[new_dim]], output_core_dims=[[new_dim]],
vectorize=True, kwargs={"h": h}, output_dtypes=[d.dtype])
@@ -774,13 +765,13 @@ class ClimateFIRFilter(FIRFilter):
gc.collect()
# concat all variables
- logging.debug(f"{station_name}: concat all variables")
+ logging.debug(f"{display_name}: concat all variables")
res = xr.concat(coll, var_dim)
res_input = xr.concat(coll_input, var_dim)
# create result array with same shape like input data, gaps are filled by nans
- res_full = self._create_full_filter_result_array(data, res, new_dim, station_name)
- res_input_full = self._create_full_filter_result_array(data, res_input, new_dim, station_name)
+ res_full = self._create_full_filter_result_array(data, res, new_dim, display_name)
+ res_input_full = self._create_full_filter_result_array(data, res_input, new_dim, display_name)
return res_full, res_input_full, h, apriori, plot_data
@staticmethod
diff --git a/mlair/plotting/data_insight_plotting.py b/mlair/plotting/data_insight_plotting.py
index 2048f4c2..526ee867 100644
--- a/mlair/plotting/data_insight_plotting.py
+++ b/mlair/plotting/data_insight_plotting.py
@@ -977,42 +977,37 @@ class PlotClimateFirFilter(AbstractPlotClass): # pragma: no cover
for p_d in plot_data:
var = p_d.get("var")
t0 = p_d.get("t0")
- viz_date = p_d.get("viz_date")
filter_input = p_d.get("filter_input")
filter_input_nc = p_d.get("filter_input_nc")
valid_range = p_d.get("valid_range")
time_range = p_d.get("time_range")
- offset = p_d.get("offset")
if new_dim is None:
new_dim = p_d.get("new_dim")
else:
assert new_dim == p_d.get("new_dim")
h = p_d.get("h")
plot_dict_var = plot_dict.get(var, {})
- plot_dict_t0 = plot_dict_var.get(viz_date, {})
+ plot_dict_t0 = plot_dict_var.get(t0, {})
plot_dict_order = {"filter_input": filter_input,
"filter_input_nc": filter_input_nc,
"valid_range": valid_range,
"time_range": time_range,
- "order": len(h), "h": h,
- "t0": t0,
- "offset": offset}
+ "order": len(h), "h": h}
plot_dict_t0[i] = plot_dict_order
- plot_dict_var[viz_date] = plot_dict_t0
+ plot_dict_var[t0] = plot_dict_t0
plot_dict[var] = plot_dict_var
self.variables_list = list(plot_dict.keys())
return plot_dict, new_dim
def _plot(self, plot_dict, sampling, new_dim="window"):
td_type = {"1d": "D", "1H": "h"}.get(sampling)
- for var, viz_date_dict in plot_dict.items():
- for iviz_date, viz_date in enumerate(viz_date_dict.keys()):
- viz_data = viz_date_dict[viz_date]
+ for var, vis_dict in plot_dict.items():
+ for it0, t0 in enumerate(vis_dict.keys()):
+ vis_data = vis_dict[t0]
residuum_true = None
try:
- for ifilter in sorted(viz_data.keys()):
- data = viz_data[ifilter]
- t0 = data["t0"]
+ for ifilter in sorted(vis_data.keys()):
+ data = vis_data[ifilter]
filter_input = data["filter_input"]
filter_input_nc = data["filter_input_nc"] if residuum_true is None else residuum_true.sel(
{new_dim: filter_input.coords[new_dim]})
@@ -1020,11 +1015,10 @@ class PlotClimateFirFilter(AbstractPlotClass): # pragma: no cover
time_axis = data["time_range"]
filter_order = data["order"]
h = data["h"]
- offset = data["offset"]
fig, ax = plt.subplots()
# plot backgrounds
- self._plot_valid_area(ax, viz_date, valid_range, td_type)
+ self._plot_valid_area(ax, t0, valid_range, td_type)
self._plot_t0(ax, t0)
# original data
@@ -1048,12 +1042,12 @@ class PlotClimateFirFilter(AbstractPlotClass): # pragma: no cover
plt.legend()
fig.autofmt_xdate()
plt.tight_layout()
- self.plot_name = f"climFIR_{self._name}_{str(var)}_{iviz_date}_{ifilter}"
+ self.plot_name = f"climFIR_{self._name}_{str(var)}_{it0}_{ifilter}"
self._save()
# plot residuum
fig, ax = plt.subplots()
- self._plot_valid_area(ax, viz_date, valid_range, td_type)
+ self._plot_valid_area(ax, t0, valid_range, td_type)
self._plot_t0(ax, t0)
self._plot_series(ax, time_axis, residuum_true.values.flatten(), style="ideal")
self._plot_series(ax, time_axis, residuum_estimated.values.flatten(), style="clim")
@@ -1063,7 +1057,7 @@ class PlotClimateFirFilter(AbstractPlotClass): # pragma: no cover
fig.autofmt_xdate()
plt.tight_layout()
- self.plot_name = f"climFIR_{self._name}_{str(var)}_{iviz_date}_{ifilter}_residuum"
+ self.plot_name = f"climFIR_{self._name}_{str(var)}_{it0}_{ifilter}_residuum"
self._save()
except Exception as e:
logging.info(f"Could not create plot because of:\n{sys.exc_info()[0]}\n{sys.exc_info()[1]}\n{sys.exc_info()[2]}")
diff --git a/test/test_helpers/test_filter.py b/test/test_helpers/test_filter.py
index 6f72eda6..3c362911 100644
--- a/test/test_helpers/test_filter.py
+++ b/test/test_helpers/test_filter.py
@@ -132,7 +132,7 @@ class TestClimateFIRFilter:
res = obj._next_order([128, 64, 43], None, 0, "hamming")
assert res == 64
res = obj._next_order([43], None, 0, "hamming")
- assert res is None
+ assert res == 0
def test_next_order_with_kzf(self):
obj = object.__new__(ClimateFIRFilter)
@@ -315,28 +315,29 @@ class TestClimateFIRFilter:
def test_clim_filter(self, xr_array_long_with_var, time_dim, var_dim):
obj = object.__new__(ClimateFIRFilter)
filter_order = 10*24+1
- res = obj.clim_filter(xr_array_long_with_var, 24, 0.05, 10*24+1, sampling="1H", time_dim=time_dim, var_dim=var_dim)
+ res = obj.clim_filter(xr_array_long_with_var, 24, 0.05, filter_order, sampling="1H", time_dim=time_dim,
+ var_dim=var_dim, minimum_length=24)
assert len(res) == 5
# check filter data properties
- assert res[0].shape == (*xr_array_long_with_var.shape, filter_order + 1)
+ assert res[0].shape == (*xr_array_long_with_var.shape, 24 + 2)
assert res[0].dims == (*xr_array_long_with_var.dims, "window")
# check filter properties
assert np.testing.assert_almost_equal(
- res[1], obj._calculate_filter_coefficients("hamming", filter_order, 0.05, 24)) is None
+ res[2], obj._calculate_filter_coefficients("hamming", filter_order, 0.05, 24)) is None
# check apriori
apriori = obj.create_monthly_mean(xr_array_long_with_var, time_dim, sampling="1H")
apriori = apriori.astype(xr_array_long_with_var.dtype)
apriori = obj.extend_apriori(xr_array_long_with_var, apriori, time_dim, "1H")
- assert xr.testing.assert_equal(res[2], apriori) is None
+ assert xr.testing.assert_equal(res[3], apriori) is None
# check plot data format
- assert isinstance(res[3], list)
- assert isinstance(res[3][0], dict)
+ assert isinstance(res[4], list)
+ assert isinstance(res[4][0], dict)
keys = {"t0", "var", "filter_input", "filter_input_nc", "valid_range", "time_range", "h", "new_dim"}
- assert len(keys.symmetric_difference(res[3][0].keys())) == 0
+ assert len(keys.symmetric_difference(res[4][0].keys())) == 0
def test_clim_filter_kwargs(self, xr_array_long_with_var, time_dim, var_dim):
obj = object.__new__(ClimateFIRFilter)
@@ -352,9 +353,9 @@ class TestClimateFIRFilter:
assert res[0].shape == (*xr_array_long_with_var.shape, 1000 + 2)
assert res[0].dims == (*xr_array_long_with_var.dims, "total_new_dim")
assert np.testing.assert_almost_equal(
- res[1], obj._calculate_filter_coefficients(("kaiser", 5), filter_order, 0.05, 24)) is None
- assert xr.testing.assert_equal(res[2], apriori) is None
- assert len(res[3]) == len(res[0].coords[var_dim])
+ res[2], obj._calculate_filter_coefficients(("kaiser", 5), filter_order, 0.05, 24)) is None
+ assert xr.testing.assert_equal(res[3], apriori) is None
+ assert len(res[4]) == len(res[0].coords[var_dim])
class TestFirFilterConvolve:
--
GitLab