Skip to content
Snippets Groups Projects
Commit 5d65e422 authored by leufen1's avatar leufen1
Browse files

filter can now handle ifs forecast data, some validation check are recommended

parent 4cfa8a11
Branches
Tags
3 merge requests!522filter can now combine obs, forecast, and apriori for first iteration. Further...,!521Resolve "release v2.4.0",!517Resolve "load ifs data"
Pipeline #142371 failed
......@@ -270,12 +270,13 @@ class ClimateFIRFilter(FIRFilter):
filtered.append(fi.sel({new_dim: slice(self.extend_end - self.minimum_length, self.extend_end)}))
h.append(hi)
gc.collect()
forecasts = None # reset/disable forecasts after first filter iteration
self.plot_data.append(plot_data)
plot_dates = {e["t0"] for e in plot_data}
# calculate residuum
logging.info(f"{self.display_name}: calculate residuum")
coord_range = range(fi.coords[new_dim].values.min(), fi.coords[new_dim].values.max() + 1)
coord_range = range(int(fi.coords[new_dim].values.min()), int(fi.coords[new_dim].values.max() + 1))
if new_dim in input_data.coords:
input_data = input_data.sel({new_dim: coord_range}) - fi
else:
......@@ -575,7 +576,7 @@ class ClimateFIRFilter(FIRFilter):
"""
# prepare historical data / observation
if forecasts is None:
ext_sep = offset + min(extend_end, extend_length_future)
ext_sep = offset + min(extend_end, extend_length_future, extend_length_separator)
else:
ext_sep = min(offset, extend_length_future)
if new_dim not in data.coords:
......@@ -586,9 +587,10 @@ class ClimateFIRFilter(FIRFilter):
if forecasts is not None:
forecast_delta = self.get_forecast_run_delta(forecasts, time_dim)
forecast_end = offset + min(extend_end, extend_length_separator)
for lead_time in forecasts.coords[new_dim]:
delta = np.timedelta64(int(lead_time - offset), {"1d": "D", "1H": "h"}.get(sampling))
forecasts_tmp = forecasts.sel({new_dim: slice(None, extend_end)})
forecasts_tmp = forecasts.sel({new_dim: slice(None, forecast_end)})
forecasts_tmp.coords[time_dim] = forecasts_tmp.coords[time_dim] + delta
forecasts_tmp.coords[new_dim] = forecasts_tmp.coords[new_dim] + offset - lead_time
history = history.combine_first(forecasts_tmp)
......@@ -612,14 +614,14 @@ class ClimateFIRFilter(FIRFilter):
# prepare climatological statistics
if new_dim not in apriori.coords:
future = self._shift_data(apriori, range(ext_sep + 1,
extend_length_future + 1),
offset + extend_length_future + 1),
time_dim, new_dim)
else:
future = apriori.sel({new_dim: slice(ext_sep + 1, extend_length_future)})
future = apriori.sel({new_dim: slice(ext_sep + 1, offset + extend_length_future)})
# combine historical data [t0-length,t0+sep] and climatological statistics [t0+sep+1,t0+length]
filter_input_data = history.combine_first(future)
# filter_input_data = xr.concat([history.dropna(time_dim), future], dim=new_dim, join="left")
return filter_input_data
return filter_input_data.dropna(time_dim)
else:
return history
......@@ -717,19 +719,18 @@ class ClimateFIRFilter(FIRFilter):
@staticmethod
def _trim_data_to_minimum_length(data: xr.DataArray, extend_length_history: int, dim: str,
extend_length_future: int = 0) -> xr.DataArray:
extend_length_future: int = 0, offset: int = 0) -> xr.DataArray:
"""
Trim data along given axis between either -minimum_length (if given) or -extend_length_history and
extend_length_opts (which is default set to 0).
:param data: data to trim
:param extend_length_history: start number for trim range (transformed to negative), only used if parameter
minimum_length is not provided
:param extend_length_history: start number for trim range, only used if parameter minimum_length is not provided
:param dim: dim to apply trim on
:param extend_length_future: number to use in "future"
:returns: trimmed data
"""
return data.sel({dim: slice(-extend_length_history, extend_length_future)}, drop=True)
return data.sel({dim: slice(extend_length_history + offset, extend_length_future + offset)}, drop=True)
@staticmethod
def _create_full_filter_result_array(template_array: xr.DataArray, result_array: xr.DataArray, new_dim: str,
......@@ -796,7 +797,7 @@ class ClimateFIRFilter(FIRFilter):
_year, sampling, max(extend_length_history, extend_length_future))
d = data.sel({var_dim: [var], time_dim: time_slice})
a = apriori.sel({var_dim: [var], time_dim: time_slice})
f = forecasts.sel({var_dim: [var]})
f = forecasts.sel({var_dim: [var]}) if forecasts is not None else None
if len(d.coords[time_dim]) == 0: # no data at all for this year
continue
......@@ -821,12 +822,13 @@ class ClimateFIRFilter(FIRFilter):
vectorize=True, kwargs={"h": h}, output_dtypes=[d.dtype])
# trim data if required
valid_range_start = int(filt.coords[new_dim].min() + (length + 1) / 2)
valid_range_end = int(filt.coords[new_dim].max() - (length + 1) / 2) + 1
ext_len = min(extend_length_future, valid_range_end) #todo start here and adjust by offset parameter
trimmed = self._trim_data_to_minimum_length(filt, extend_length_history, new_dim,
ext_len = min(extend_length_future, valid_range_end)
trimmed = self._trim_data_to_minimum_length(filt, valid_range_start, new_dim,
extend_length_future=ext_len)
filt_coll.append(trimmed)
trimmed = self._trim_data_to_minimum_length(filter_input_data, extend_length_history, new_dim,
trimmed = self._trim_data_to_minimum_length(filter_input_data, valid_range_start, new_dim,
extend_length_future=ext_len)
filt_input_coll.append(trimmed)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment