diff --git a/mlair/helpers/filter.py b/mlair/helpers/filter.py index 1ba0a914c7dac4454af406e405ae5178049ee336..370c8b754def06c467fabf0cf8007cada3f1a4f9 100644 --- a/mlair/helpers/filter.py +++ b/mlair/helpers/filter.py @@ -270,12 +270,13 @@ class ClimateFIRFilter(FIRFilter): filtered.append(fi.sel({new_dim: slice(self.extend_end - self.minimum_length, self.extend_end)})) h.append(hi) gc.collect() + forecasts = None # reset/disable forecasts after first filter iteration self.plot_data.append(plot_data) plot_dates = {e["t0"] for e in plot_data} # calculate residuum logging.info(f"{self.display_name}: calculate residuum") - coord_range = range(fi.coords[new_dim].values.min(), fi.coords[new_dim].values.max() + 1) + coord_range = range(int(fi.coords[new_dim].values.min()), int(fi.coords[new_dim].values.max() + 1)) if new_dim in input_data.coords: input_data = input_data.sel({new_dim: coord_range}) - fi else: @@ -575,7 +576,7 @@ class ClimateFIRFilter(FIRFilter): """ # prepare historical data / observation if forecasts is None: - ext_sep = offset + min(extend_end, extend_length_future) + ext_sep = offset + min(extend_end, extend_length_future, extend_length_separator) else: ext_sep = min(offset, extend_length_future) if new_dim not in data.coords: @@ -586,9 +587,10 @@ class ClimateFIRFilter(FIRFilter): if forecasts is not None: forecast_delta = self.get_forecast_run_delta(forecasts, time_dim) + forecast_end = offset + min(extend_end, extend_length_separator) for lead_time in forecasts.coords[new_dim]: delta = np.timedelta64(int(lead_time - offset), {"1d": "D", "1H": "h"}.get(sampling)) - forecasts_tmp = forecasts.sel({new_dim: slice(None, extend_end)}) + forecasts_tmp = forecasts.sel({new_dim: slice(None, forecast_end)}) forecasts_tmp.coords[time_dim] = forecasts_tmp.coords[time_dim] + delta forecasts_tmp.coords[new_dim] = forecasts_tmp.coords[new_dim] + offset - lead_time history = history.combine_first(forecasts_tmp) @@ -612,14 +614,14 @@ class ClimateFIRFilter(FIRFilter): # prepare climatological statistics if new_dim not in apriori.coords: future = self._shift_data(apriori, range(ext_sep + 1, - extend_length_future + 1), + offset + extend_length_future + 1), time_dim, new_dim) else: - future = apriori.sel({new_dim: slice(ext_sep + 1, extend_length_future)}) + future = apriori.sel({new_dim: slice(ext_sep + 1, offset + extend_length_future)}) # combine historical data [t0-length,t0+sep] and climatological statistics [t0+sep+1,t0+length] filter_input_data = history.combine_first(future) # filter_input_data = xr.concat([history.dropna(time_dim), future], dim=new_dim, join="left") - return filter_input_data + return filter_input_data.dropna(time_dim) else: return history @@ -717,19 +719,18 @@ class ClimateFIRFilter(FIRFilter): @staticmethod def _trim_data_to_minimum_length(data: xr.DataArray, extend_length_history: int, dim: str, - extend_length_future: int = 0) -> xr.DataArray: + extend_length_future: int = 0, offset: int = 0) -> xr.DataArray: """ Trim data along given axis between either -minimum_length (if given) or -extend_length_history and extend_length_opts (which is default set to 0). :param data: data to trim - :param extend_length_history: start number for trim range (transformed to negative), only used if parameter - minimum_length is not provided + :param extend_length_history: start number for trim range, only used if parameter minimum_length is not provided :param dim: dim to apply trim on :param extend_length_future: number to use in "future" :returns: trimmed data """ - return data.sel({dim: slice(-extend_length_history, extend_length_future)}, drop=True) + return data.sel({dim: slice(extend_length_history + offset, extend_length_future + offset)}, drop=True) @staticmethod def _create_full_filter_result_array(template_array: xr.DataArray, result_array: xr.DataArray, new_dim: str, @@ -796,7 +797,7 @@ class ClimateFIRFilter(FIRFilter): _year, sampling, max(extend_length_history, extend_length_future)) d = data.sel({var_dim: [var], time_dim: time_slice}) a = apriori.sel({var_dim: [var], time_dim: time_slice}) - f = forecasts.sel({var_dim: [var]}) + f = forecasts.sel({var_dim: [var]}) if forecasts is not None else None if len(d.coords[time_dim]) == 0: # no data at all for this year continue @@ -821,12 +822,13 @@ class ClimateFIRFilter(FIRFilter): vectorize=True, kwargs={"h": h}, output_dtypes=[d.dtype]) # trim data if required + valid_range_start = int(filt.coords[new_dim].min() + (length + 1) / 2) valid_range_end = int(filt.coords[new_dim].max() - (length + 1) / 2) + 1 - ext_len = min(extend_length_future, valid_range_end) #todo start here and adjust by offset parameter - trimmed = self._trim_data_to_minimum_length(filt, extend_length_history, new_dim, + ext_len = min(extend_length_future, valid_range_end) + trimmed = self._trim_data_to_minimum_length(filt, valid_range_start, new_dim, extend_length_future=ext_len) filt_coll.append(trimmed) - trimmed = self._trim_data_to_minimum_length(filter_input_data, extend_length_history, new_dim, + trimmed = self._trim_data_to_minimum_length(filter_input_data, valid_range_start, new_dim, extend_length_future=ext_len) filt_input_coll.append(trimmed)