diff --git a/src/data_handling/bootstraps.py b/src/data_handling/bootstraps.py
index ba1234f84148fb2f28ec8df2bfa5846900881eb5..f68532dac279994b235514bcdec0cf9f6decfdc1 100644
--- a/src/data_handling/bootstraps.py
+++ b/src/data_handling/bootstraps.py
@@ -10,7 +10,7 @@ import xarray as xr
import os
import re
from src import helpers
-from typing import List, Union
+from typing import List, Union, Pattern
class BootStrapGenerator:
@@ -39,7 +39,8 @@ class BootStrapGenerator:
for boot in range(self.number_of_boots):
logging.debug(f"boot: {boot}")
boot_hist = hist.sel(variables=helpers.list_pop(self.variables, var))
- shuffled_var = shuffled_data.sel(variables=var, boots=boot).expand_dims("variables").drop("boots").transpose("datetime", "window", "Stations", "variables")
+ shuffled_var = shuffled_data.sel(variables=var, boots=boot).expand_dims("variables").drop("boots")
+ shuffled_var = shuffled_var.transpose("datetime", "window", "Stations", "variables")
boot_hist = boot_hist.combine_first(shuffled_var)
boot_hist = boot_hist.sortby("variables")
yield boot_hist
@@ -90,18 +91,52 @@ class BootStrapGenerator:
:param variables:
:return: shuffled data as xarray
"""
- files = os.listdir(self.bootstrap_path)
- regex = self.create_file_regex(station, variables)
- file_name = os.path.join(self.bootstrap_path, list(filter(regex.search, files))[0])
+ file_name = self.get_shuffled_data_file(station, variables)
shuffled_data = xr.open_dataarray(file_name, chunks=100)
return shuffled_data
+ def get_shuffled_data_file(self, station, variables):
+ files = os.listdir(self.bootstrap_path)
+ regex = self.create_file_regex(station, variables)
+ file = self.filter_files(regex, files, self.orig_generator.window_history_size, self.number_of_boots)
+ if file:
+ return os.path.join(self.bootstrap_path, file)
+ else:
+ raise FileNotFoundError(f"Could not find a file to match pattern {regex}")
+
@staticmethod
- def create_file_regex(station, variables):
- var_regex = "".join([rf'(_\w+)*_{v}(_\w+)*' for v in sorted(variables)])
- regex = re.compile(rf"{station}{var_regex}_shuffled\.nc")
+ def create_file_regex(station: str, variables: List[str]) -> Pattern:
+ """
+ Creates regex for given station and variables to look for shuffled data with pattern:
+ `<station>(_<var>)*_hist(<hist>)_nboots(<nboots>)_shuffled.nc`
+ :param station: station name to use as prefix
+ :param variables: variables to add after station
+ :return: compiled regular expression
+ """
+ var_regex = "".join([rf"(_\w+)*_{v}(_\w+)*" for v in sorted(variables)])
+ regex = re.compile(rf"{station}{var_regex}_hist(\d+)_nboots(\d+)_shuffled\.nc")
return regex
+ @staticmethod
+ def filter_files(regex: Pattern, files: List[str], window: int, nboot: int) -> Union[str, None]:
+ """
+ Filter list of files by regex. Regex has to be structured to match the following string structure
+ `<station>(_<var>)*_hist(<hist>)_nboots(<nboots>)_shuffled.nc`. Hist and nboots values have to be included as
+ group. All matches are compared to given window and nboot parameters. A valid file must have the same value (or
+ larger) than these parameters and contain all variables.
+ :param regex: compiled regular expression pattern following the style from method description
+ :param files: list of file names to filter
+ :param window: minimum length of window to look for
+ :param nboot: minimal number of boots to search
+ :return: matching file name or None, if no valid file was found
+ """
+ for f in files:
+ match = regex.match(f)
+ if match:
+ last = match.lastindex
+ if (int(match.group(last-1)) >= window) and (int(match.group(last)) >= nboot):
+ return f
+
class BootStraps:
@@ -189,7 +224,7 @@ class BootStraps:
length is ge than given window size form args and the number of boots is also ge than the given number of boots
from this class. Furthermore, this functions deletes local files, if the match the station pattern but don't fit
the window and bootstrap condition. This is performed, because it is assumed, that the corresponding file will
- be created with a longer or at least same window size and numbers of bootstraps.
+ be created with a longer or at the least same window size and numbers of bootstraps.
:param station:
:param variables:
:param window:
diff --git a/src/run_modules/post_processing.py b/src/run_modules/post_processing.py
index 3d4238b8b25ee432feea0f8c0dcc009d60129ad5..21c73e8ded2b47362874ee272983a89311bf9b6b 100644
--- a/src/run_modules/post_processing.py
+++ b/src/run_modules/post_processing.py
@@ -132,7 +132,7 @@ class PostProcessing(RunEnvironment):
labels = xr.DataArray(labels, coords=(*coords, ["obs"]), dims=["index", "ahead", "type"])
labels.to_netcdf(file_name)
- def calculate_bootstrap_skill_scores(self) -> Dict[xr.DataArray]:
+ def calculate_bootstrap_skill_scores(self) -> Dict[str, xr.DataArray]:
"""
Use already created bootstrap predictions and the original predictions (the not-bootstrapped ones) and calculate
skill scores for the bootstraps. The result is saved as a xarray DataArray in a dictionary structure separated
diff --git a/test/test_data_handling/test_bootstraps.py b/test/test_data_handling/test_bootstraps.py
index 00530bf4d025126c527085454ee6f7a3550ad997..3c2cac109e09b4a1871230bd2af6afd7901c98a2 100644
--- a/test/test_data_handling/test_bootstraps.py
+++ b/test/test_data_handling/test_bootstraps.py
@@ -2,8 +2,9 @@
from src.data_handling.bootstraps import BootStraps, BootStrapGenerator
from src.data_handling.data_generator import DataGenerator
-import pytest
import os
+import pytest
+import typing
import numpy as np
import xarray as xr
@@ -71,14 +72,15 @@ class TestBootstrapGenerator:
@pytest.fixture
def orig_generator(self):
return DataGenerator(os.path.join(os.path.dirname(__file__), 'data'), 'AIRBASE', ['DEBW107', 'DEBW013'],
- ['o3', 'temp'], 'datetime', 'variables', 'o3', start=2010, end=2014)
+ ['o3', 'temp'], 'datetime', 'variables', 'o3', start=2010, end=2014,
+ statistics_per_var={"o3": "dma8eu", "temp": "maximum"})
@pytest.fixture
def boot_gen(self, orig_generator):
path = os.path.join(os.path.dirname(__file__), 'data')
dummy_content = xr.DataArray([1, 2, 3], dims="dummy")
- dummy_content.to_netcdf(os.path.join(path, "DEBW107_o3_temp_shuffled.nc"))
- dummy_content.to_netcdf(os.path.join(path, "DEBW013_o3_temp_shuffled.nc"))
+ dummy_content.to_netcdf(os.path.join(path, "DEBW107_o3_temp_hist7_nboots20_shuffled.nc"))
+ dummy_content.to_netcdf(os.path.join(path, "DEBW013_o3_temp_hist7_nboots20_shuffled.nc"))
return BootStrapGenerator(orig_generator, 20, path)
def test_init(self, orig_generator):
@@ -91,14 +93,35 @@ class TestBootstrapGenerator:
def test_len(self, boot_gen):
assert len(boot_gen) == 80
- def test_get_generator(self, boot_gen):
- pass
-
- def test_get_bootstrap_meta(self, boot_gen):
- pass
+ def test_get_generator_station_var_wise(self, boot_gen, orig_generator):
+ res = boot_gen.get_generator_station_var_wise("DEBW107", "o3")
+ hist = orig_generator.get_data_generator("DEBW107").get_transposed_history()
+ assert xr.testing.assert_equal(res[0], hist) is None
+ label = orig_generator.get_data_generator("DEBW107").get_transposed_label()
+ assert xr.testing.assert_equal(res[1], label) is None
+ assert isinstance(res[2], typing.Callable)
+ assert res[3] == 20
+
+ def test_get_bootstrap_station_var_wise_meta(self, boot_gen):
+ meta = boot_gen.get_bootstrap_meta_station_var_wise("DEBW107", "o3")
+ labels = boot_gen.orig_generator.get_data_generator("DEBW107").get_transposed_label().shape[0]
+ assert np.shape(meta) == (labels * boot_gen.number_of_boots, 2)
+ assert np.testing.assert_array_equal(np.unique(meta), ["DEBW107", "o3"])
def test_get_labels(self, boot_gen):
- pass
+ res = []
+ for label in boot_gen.get_labels("DEBW107"):
+ res.append(label)
+ assert len(res) == boot_gen.number_of_boots
+ assert xr.testing.assert_equal(res[0], res[-1]) is None
+
+ def all_equal(check_list):
+ equal = True
+ for b in check_list:
+ equal *= xr.testing.assert_equal(check_list[0], b) is None
+ return equal
+ assert all_equal(res)
+
def test_get_orig_prediction(self, boot_gen):
pass
@@ -108,12 +131,25 @@ class TestBootstrapGenerator:
assert isinstance(shuffled_data, xr.DataArray)
assert all(shuffled_data.compute().values == [1, 2, 3])
+ def test_get_shuffled_data_file(self, boot_gen):
+ file_name = boot_gen.get_shuffled_data_file("DEBW107", ["o3"])
+ assert file_name == os.path.join(boot_gen.bootstrap_path, "DEBW107_o3_temp_hist7_nboots20_shuffled.nc")
+
+ def test_get_shuffled_data_file_not_found(self, boot_gen):
+ boot_gen.number_of_boots = 100
+ with pytest.raises(FileNotFoundError) as e:
+ boot_gen.get_shuffled_data_file("DEBW107", ["o3"])
+ assert "Could not find a file to match pattern" in e.value.args[0]
+
def test_create_file_regex(self, boot_gen):
regex = boot_gen.create_file_regex("DEBW108", ["o3", "temp", "h2o"])
test_list = ["DEBW108_o3_test23_test_shuffled.nc",
"DEBW107_o3_test23_test_shuffled.nc",
"DEBW108_o3_test23_test.nc",
"DEBW108_h2o_o3_temp_test_shuffled.nc",
- "DEBW108_h2o_hum_latent_o3_temp_u_v_test23_test_shuffled.nc"]
- assert list(filter(regex.search, test_list)) == ["DEBW108_h2o_o3_temp_test_shuffled.nc",
- "DEBW108_h2o_hum_latent_o3_temp_u_v_test23_test_shuffled.nc"]
+ "DEBW108_h2o_hum_latent_o3_temp_u_v_test23_test_shuffled.nc",
+ "DEBW108_o3_temp_hist9_nboots20_shuffled.nc",
+ "DEBW108_h2o_o3_temp_hist9_nboots20_shuffled.nc"]
+ assert boot_gen.filter_files(regex, test_list, 10, 10) is None
+ assert boot_gen.filter_files(regex, test_list, 9, 10) == "DEBW108_h2o_o3_temp_hist9_nboots20_shuffled.nc"
+ assert boot_gen.filter_files(regex, test_list, 9, 20) == "DEBW108_h2o_o3_temp_hist9_nboots20_shuffled.nc"