Skip to content
Snippets Groups Projects
Commit 69ffde01 authored by leufen1's avatar leufen1
Browse files

remove 1 station from preproc test to accelerate

parent 4708ee98
Branches
Tags
5 merge requests!480Merge multiple stats into crps working branch,!470Develop,!467Resolve "release v2.2.0",!466Draft: Resolve "Include CRPS analysis and other ens verif methods or plots",!460Resolve "TECH: reduce CI running time"
Pipeline #106876 failed
...@@ -28,7 +28,7 @@ class TestPreProcessing: ...@@ -28,7 +28,7 @@ class TestPreProcessing:
@pytest.fixture @pytest.fixture
def obj_with_exp_setup(self): def obj_with_exp_setup(self):
ExperimentSetup(stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBW99X'], ExperimentSetup(stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW087', 'DEBW99X'],
statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'}, station_type="background", statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'}, station_type="background",
data_handler=DefaultDataHandler) data_handler=DefaultDataHandler)
pre = object.__new__(PreProcessing) pre = object.__new__(PreProcessing)
...@@ -37,15 +37,15 @@ class TestPreProcessing: ...@@ -37,15 +37,15 @@ class TestPreProcessing:
RunEnvironment().__del__() RunEnvironment().__del__()
def test_init(self, caplog): def test_init(self, caplog):
ExperimentSetup(stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'], ExperimentSetup(stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW087'],
statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'}) statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'})
caplog.clear() caplog.clear()
caplog.set_level(logging.INFO) caplog.set_level(logging.INFO)
with PreProcessing(): with PreProcessing():
assert caplog.record_tuples[0] == ('root', 20, 'PreProcessing started') assert caplog.record_tuples[0] == ('root', 20, 'PreProcessing started')
assert caplog.record_tuples[1] == ('root', 20, 'check valid stations started (preprocessing)') assert caplog.record_tuples[1] == ('root', 20, 'check valid stations started (preprocessing)')
assert caplog.record_tuples[-6] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 5 ' assert caplog.record_tuples[-6] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 4 '
r'station\(s\). Found 5/5 valid stations.')) r'station\(s\). Found 4/4 valid stations.'))
assert caplog.record_tuples[-5] == ('root', 20, "use serial create_info_df (train)") assert caplog.record_tuples[-5] == ('root', 20, "use serial create_info_df (train)")
assert caplog.record_tuples[-4] == ('root', 20, "use serial create_info_df (val)") assert caplog.record_tuples[-4] == ('root', 20, "use serial create_info_df (val)")
assert caplog.record_tuples[-3] == ('root', 20, "use serial create_info_df (test)") assert caplog.record_tuples[-3] == ('root', 20, "use serial create_info_df (test)")
...@@ -85,13 +85,13 @@ class TestPreProcessing: ...@@ -85,13 +85,13 @@ class TestPreProcessing:
def test_create_set_split_all_stations(self, caplog, obj_with_exp_setup): def test_create_set_split_all_stations(self, caplog, obj_with_exp_setup):
caplog.set_level(logging.DEBUG) caplog.set_level(logging.DEBUG)
obj_with_exp_setup.create_set_split(slice(0, 2), "awesome") obj_with_exp_setup.create_set_split(slice(0, 2), "awesome")
message = "Awesome stations (len=6): ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBW99X']" message = "Awesome stations (len=5): ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW99X']"
assert ('root', 10, message) in caplog.record_tuples assert ('root', 10, message) in caplog.record_tuples
data_store = obj_with_exp_setup.data_store data_store = obj_with_exp_setup.data_store
assert isinstance(data_store.get("data_collection", "general.awesome"), DataCollection) assert isinstance(data_store.get("data_collection", "general.awesome"), DataCollection)
with pytest.raises(NameNotFoundInScope): with pytest.raises(NameNotFoundInScope):
data_store.get("data_collection", "general") data_store.get("data_collection", "general")
assert data_store.get("stations", "general.awesome") == ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'] assert data_store.get("stations", "general.awesome") == ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076']
@pytest.mark.parametrize("name", (None, "tester")) @pytest.mark.parametrize("name", (None, "tester"))
def test_validate_station_serial(self, caplog, obj_with_exp_setup, name): def test_validate_station_serial(self, caplog, obj_with_exp_setup, name):
...@@ -106,8 +106,8 @@ class TestPreProcessing: ...@@ -106,8 +106,8 @@ class TestPreProcessing:
expected = "check valid stations started" + ' (%s)' % (name if name else 'all') expected = "check valid stations started" + ' (%s)' % (name if name else 'all')
assert caplog.record_tuples[0] == ('root', 20, expected) assert caplog.record_tuples[0] == ('root', 20, expected)
assert caplog.record_tuples[1] == ('root', 20, "use serial validate station approach") assert caplog.record_tuples[1] == ('root', 20, "use serial validate station approach")
assert caplog.record_tuples[-1] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 6 ' assert caplog.record_tuples[-1] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 5 '
r'station\(s\). Found 5/6 valid stations.')) r'station\(s\). Found 4/5 valid stations.'))
@mock.patch("psutil.cpu_count", return_value=3) @mock.patch("psutil.cpu_count", return_value=3)
@mock.patch("multiprocessing.Pool", return_value=multiprocessing.Pool(3)) @mock.patch("multiprocessing.Pool", return_value=multiprocessing.Pool(3))
...@@ -124,8 +124,8 @@ class TestPreProcessing: ...@@ -124,8 +124,8 @@ class TestPreProcessing:
assert caplog.record_tuples[0] == ('root', 20, "check valid stations started (all)") assert caplog.record_tuples[0] == ('root', 20, "check valid stations started (all)")
assert caplog.record_tuples[1] == ('root', 20, "use parallel validate station approach") assert caplog.record_tuples[1] == ('root', 20, "use parallel validate station approach")
assert caplog.record_tuples[2] == ('root', 20, "running 3 processes in parallel") assert caplog.record_tuples[2] == ('root', 20, "running 3 processes in parallel")
assert caplog.record_tuples[-1] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 6 ' assert caplog.record_tuples[-1] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 5 '
r'station\(s\). Found 5/6 valid stations.')) r'station\(s\). Found 4/5 valid stations.'))
def test_split_set_indices(self, obj_super_init): def test_split_set_indices(self, obj_super_init):
dummy_list = list(range(0, 15)) dummy_list = list(range(0, 15))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment