From 7f6e784524c41aba8bb0627f9d916695c668f127 Mon Sep 17 00:00:00 2001
From: lukas leufen <l.leufen@fz-juelich.de>
Date: Thu, 16 Jul 2020 16:13:06 +0200
Subject: [PATCH] updated tests (not all finished yet)
---
src/data_handling/iterator.py | 2 +-
src/model_modules/model_class.py | 12 ++--
src/run_modules/experiment_setup.py | 3 +-
src/run_modules/model_setup.py | 6 +-
test/test_data_handling/test_iterator.py | 35 ++++++++--
test/test_model_modules/test_model_class.py | 14 ++--
test/test_modules/test_model_setup.py | 73 +++++++++++++++------
test/test_modules/test_pre_processing.py | 2 +-
8 files changed, 107 insertions(+), 40 deletions(-)
diff --git a/src/data_handling/iterator.py b/src/data_handling/iterator.py
index 883ba4df..0eec4326 100644
--- a/src/data_handling/iterator.py
+++ b/src/data_handling/iterator.py
@@ -57,7 +57,7 @@ class KerasIterator(keras.utils.Sequence):
def __init__(self, collection: DataCollection, batch_size: int, batch_path: str, shuffle_batches: bool = False,
model=None, upsampling=False, name=None):
self._collection = collection
- batch_path = os.path.join(batch_path, str(name) if name is not None else id(self))
+ batch_path = os.path.join(batch_path, str(name if name is not None else id(self)))
self._path = os.path.join(batch_path, "%i.pickle")
self.batch_size = batch_size
self.model = model
diff --git a/src/model_modules/model_class.py b/src/model_modules/model_class.py
index 535941f8..ca54840c 100644
--- a/src/model_modules/model_class.py
+++ b/src/model_modules/model_class.py
@@ -341,7 +341,7 @@ class MyLittleModel(AbstractModelClass):
Dense layer.
"""
- def __init__(self, shape_inputs, shape_outputs):
+ def __init__(self, shape_inputs: list, shape_outputs: list):
"""
Sets model and loss depending on the given arguments.
@@ -350,6 +350,7 @@ class MyLittleModel(AbstractModelClass):
"""
assert len(shape_inputs) == 1
+ assert len(shape_outputs) == 1
super().__init__(shape_inputs[0], shape_outputs[0])
# settings
@@ -400,7 +401,7 @@ class MyBranchedModel(AbstractModelClass):
Dense layer.
"""
- def __init__(self, shape_inputs, shape_outputs):
+ def __init__(self, shape_inputs: list, shape_outputs: list):
"""
Sets model and loss depending on the given arguments.
@@ -409,6 +410,7 @@ class MyBranchedModel(AbstractModelClass):
"""
assert len(shape_inputs) == 1
+ assert len(shape_outputs) == 1
super().__init__(shape_inputs[0], shape_outputs[0])
# settings
@@ -457,7 +459,7 @@ class MyBranchedModel(AbstractModelClass):
class MyTowerModel(AbstractModelClass):
- def __init__(self, shape_inputs, shape_outputs):
+ def __init__(self, shape_inputs: list, shape_outputs: list):
"""
Sets model and loss depending on the given arguments.
@@ -466,6 +468,7 @@ class MyTowerModel(AbstractModelClass):
"""
assert len(shape_inputs) == 1
+ assert len(shape_outputs) == 1
super().__init__(shape_inputs[0], shape_outputs[0])
# settings
@@ -554,7 +557,7 @@ class MyTowerModel(AbstractModelClass):
class MyPaperModel(AbstractModelClass):
- def __init__(self, shape_inputs, shape_outputs):
+ def __init__(self, shape_inputs: list, shape_outputs: list):
"""
Sets model and loss depending on the given arguments.
@@ -563,6 +566,7 @@ class MyPaperModel(AbstractModelClass):
"""
assert len(shape_inputs) == 1
+ assert len(shape_outputs) == 1
super().__init__(shape_inputs[0], shape_outputs[0])
# settings
diff --git a/src/run_modules/experiment_setup.py b/src/run_modules/experiment_setup.py
index 3e471dda..0a9d7119 100644
--- a/src/run_modules/experiment_setup.py
+++ b/src/run_modules/experiment_setup.py
@@ -301,7 +301,8 @@ class ExperimentSetup(RunEnvironment):
self._set_param("sampling", sampling)
self._set_param("transformation", transformation, default=DEFAULT_TRANSFORMATION)
self._set_param("transformation", None, scope="preprocessing")
- self._set_param("data_preparation", data_preparation(), default=CustomDataClass())
+ self._set_param("data_preparation", data_preparation() if data_preparation is not None else None,
+ default=CustomDataClass())
assert isinstance(getattr(self.data_store.get("data_preparation"), "requirements"), property) is False
# target
diff --git a/src/run_modules/model_setup.py b/src/run_modules/model_setup.py
index 5acdac01..7de1c7b6 100644
--- a/src/run_modules/model_setup.py
+++ b/src/run_modules/model_setup.py
@@ -70,7 +70,7 @@ class ModelSetup(RunEnvironment):
def _run(self):
# set channels depending on inputs
- self._set_channels()
+ self._set_shapes()
# build model graph using settings from my_model_settings()
self.build_model()
@@ -88,10 +88,8 @@ class ModelSetup(RunEnvironment):
# compile model
self.compile_model()
- def _set_channels(self):
+ def _set_shapes(self):
"""Set input and output shapes from train collection."""
- # channels = list(map(lambda x: x[0].shape[-1], self.data_store.get("data_collection", "train")[0].get_X()))
- # self.data_store.set("channels", channels, self.scope)
shape = list(map(lambda x: x.shape[1:], self.data_store.get("data_collection", "train")[0].get_X()))
self.data_store.set("shape_inputs", shape, self.scope)
shape = list(map(lambda y: y.shape[1:], self.data_store.get("data_collection", "train")[0].get_Y()))
diff --git a/test/test_data_handling/test_iterator.py b/test/test_data_handling/test_iterator.py
index 3f1cf683..a4baa3ee 100644
--- a/test/test_data_handling/test_iterator.py
+++ b/test/test_data_handling/test_iterator.py
@@ -1,9 +1,11 @@
from src.data_handling.iterator import DataCollection, StandardIterator, KerasIterator
from src.helpers.testing import PyTestAllEqual
+from src.model_modules.model_class import MyLittleModel, MyBranchedModel
import numpy as np
import pytest
+import mock
import os
import shutil
@@ -56,13 +58,13 @@ class DummyData:
def __init__(self, number_of_samples=np.random.randint(100, 150)):
self.number_of_samples = number_of_samples
- def get_X(self):
+ def get_X(self, upsampling=False, as_numpy=True):
X1 = np.random.randint(0, 10, size=(self.number_of_samples, 14, 5)) # samples, window, variables
X2 = np.random.randint(21, 30, size=(self.number_of_samples, 10, 2)) # samples, window, variables
X3 = np.random.randint(-5, 0, size=(self.number_of_samples, 1, 2)) # samples, window, variables
return [X1, X2, X3]
- def get_Y(self):
+ def get_Y(self, upsampling=False, as_numpy=True):
Y1 = np.random.randint(0, 10, size=(self.number_of_samples, 5, 1)) # samples, window, variables
Y2 = np.random.randint(21, 30, size=(self.number_of_samples, 5, 1)) # samples, window, variables
return [Y1, Y2]
@@ -88,7 +90,7 @@ class TestKerasIterator:
def test_init(self, collection, path):
iterator = KerasIterator(collection, 25, path)
assert isinstance(iterator._collection, DataCollection)
- assert iterator._path == os.path.join(path, "%i.pickle")
+ assert iterator._path == os.path.join(path, str(id(iterator)), "%i.pickle")
assert iterator.batch_size == 25
assert iterator.shuffle is False
@@ -149,6 +151,8 @@ class TestKerasIterator:
iterator._collection = collection
iterator.batch_size = 50
iterator.indexes = []
+ iterator.model = None
+ iterator.upsampling = False
iterator._path = os.path.join(path, "%i.pickle")
os.makedirs(path)
iterator._prepare_batches()
@@ -162,6 +166,8 @@ class TestKerasIterator:
iterator._collection = DataCollection([DummyData(50)])
iterator.batch_size = 50
iterator.indexes = []
+ iterator.model = None
+ iterator.upsampling = False
iterator._path = os.path.join(path, "%i.pickle")
os.makedirs(path)
iterator._prepare_batches()
@@ -198,4 +204,25 @@ class TestKerasIterator:
while iterator.indexes == sorted(iterator.indexes):
iterator.on_epoch_end()
assert iterator.indexes != [0, 1, 2, 3, 4]
- assert sorted(iterator.indexes) == [0, 1, 2, 3, 4]
\ No newline at end of file
+ assert sorted(iterator.indexes) == [0, 1, 2, 3, 4]
+
+ def test_get_model_rank_no_model(self):
+ iterator = object.__new__(KerasIterator)
+ iterator.model = None
+ assert iterator._get_model_rank() == 1
+
+ def test_get_model_rank_single_output_branch(self):
+ iterator = object.__new__(KerasIterator)
+ iterator.model = MyLittleModel(shape_inputs=[(14, 1, 2)], shape_outputs=[(3,)])
+ assert iterator._get_model_rank() == 1
+
+ def test_get_model_rank_multiple_output_branch(self):
+ iterator = object.__new__(KerasIterator)
+ iterator.model = MyBranchedModel(shape_inputs=[(14, 1, 2)], shape_outputs=[(3,)])
+ assert iterator._get_model_rank() == 3
+
+ def test_get_model_rank_error(self):
+ iterator = object.__new__(KerasIterator)
+ iterator.model = mock.MagicMock(return_value=1)
+ with pytest.raises(TypeError):
+ iterator._get_model_rank()
diff --git a/test/test_model_modules/test_model_class.py b/test/test_model_modules/test_model_class.py
index 0ee2eb7e..78adb92f 100644
--- a/test/test_model_modules/test_model_class.py
+++ b/test/test_model_modules/test_model_class.py
@@ -12,7 +12,7 @@ class Paddings:
class AbstractModelSubClass(AbstractModelClass):
def __init__(self):
- super().__init__()
+ super().__init__(shape_inputs=(12, 1, 1), shape_outputs=3)
self.test_attr = "testAttr"
@@ -20,7 +20,7 @@ class TestAbstractModelClass:
@pytest.fixture
def amc(self):
- return AbstractModelClass()
+ return AbstractModelClass(shape_inputs=(14, 1, 2), shape_outputs=(3,))
@pytest.fixture
def amsc(self):
@@ -31,6 +31,8 @@ class TestAbstractModelClass:
# assert amc.loss is None
assert amc.model_name == "AbstractModelClass"
assert amc.custom_objects == {}
+ assert amc.shape_inputs == (14, 1, 2)
+ assert amc.shape_outputs == 3
def test_model_property(self, amc):
amc.model = keras.Model()
@@ -179,8 +181,10 @@ class TestAbstractModelClass:
assert amc.compile == amc.model.compile
def test_get_settings(self, amc, amsc):
- assert amc.get_settings() == {"model_name": "AbstractModelClass"}
- assert amsc.get_settings() == {"test_attr": "testAttr", "model_name": "AbstractModelSubClass"}
+ assert amc.get_settings() == {"model_name": "AbstractModelClass", "shape_inputs": (14, 1, 2),
+ "shape_outputs": 3}
+ assert amsc.get_settings() == {"test_attr": "testAttr", "model_name": "AbstractModelSubClass",
+ "shape_inputs": (12, 1, 2), "shape_outputs": 3}
def test_custom_objects(self, amc):
amc.custom_objects = {"Test": 123}
@@ -200,7 +204,7 @@ class TestMyPaperModel:
@pytest.fixture
def mpm(self):
- return MyPaperModel(window_history_size=6, window_lead_time=4, channels=9)
+ return MyPaperModel(shape_inputs=[(7, 1, 9)], shape_outputs=[(4,)])
def test_init(self, mpm):
# check if loss number of loss functions fit to model outputs
diff --git a/test/test_modules/test_model_setup.py b/test/test_modules/test_model_setup.py
index 6de61b2d..5150eade 100644
--- a/test/test_modules/test_model_setup.py
+++ b/test/test_modules/test_model_setup.py
@@ -1,9 +1,11 @@
import os
+import numpy as np
+import shutil
import pytest
-from src.data_handling import DataPrepJoin
-from src.data_handling.data_generator import DataGenerator
+from src.data_handling import KerasIterator
+from src.data_handling import DataCollection
from src.helpers.datastore import EmptyScope
from src.model_modules.keras_extensions import CallbackHandler
from src.model_modules.model_class import AbstractModelClass, MyLittleModel
@@ -29,29 +31,40 @@ class TestModelSetup:
RunEnvironment().__del__()
@pytest.fixture
- def gen(self):
- return DataGenerator(os.path.join(os.path.dirname(__file__), 'data'), 'DEBW107', ['o3', 'temp'],
- 'datetime', 'variables', 'o3', statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'},
- data_preparation=DataPrepJoin)
+ def path(self):
+ p = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testdata")
+ shutil.rmtree(p, ignore_errors=True) if os.path.exists(p) else None
+ yield p
+ shutil.rmtree(p, ignore_errors=True)
@pytest.fixture
- def setup_with_gen(self, setup, gen):
- setup.data_store.set("generator", gen, "general.train")
- setup.data_store.set("window_history_size", gen.window_history_size, "general")
- setup.data_store.set("window_lead_time", gen.window_lead_time, "general")
- setup.data_store.set("channels", 2, "general")
+ def keras_iterator(self, path):
+ coll = []
+ for i in range(3):
+ coll.append(DummyData(50 + i))
+ data_coll = DataCollection(collection=coll)
+ KerasIterator(data_coll, 25, path)
+ return data_coll
+
+ @pytest.fixture
+ def setup_with_gen(self, setup, keras_iterator):
+ setup.data_store.set("data_collection", keras_iterator, "train")
+ shape_inputs = [keras_iterator[0].get_X()[0].shape[1:]]
+ setup.data_store.set("shape_inputs", shape_inputs, "model")
+ shape_outputs = [keras_iterator[0].get_Y()[0].shape[1:]]
+ setup.data_store.set("shape_outputs", shape_outputs, "model")
yield setup
RunEnvironment().__del__()
@pytest.fixture
- def setup_with_gen_tiny(self, setup, gen):
- setup.data_store.set("generator", gen, "general.train")
+ def setup_with_gen_tiny(self, setup, keras_iterator):
+ setup.data_store.set("data_collection", keras_iterator, "train")
yield setup
RunEnvironment().__del__()
@pytest.fixture
def setup_with_model(self, setup):
- setup.model = AbstractModelClass()
+ setup.model = AbstractModelClass(shape_inputs=(12, 1), shape_outputs=2)
setup.model.test_param = "42"
yield setup
RunEnvironment().__del__()
@@ -89,14 +102,17 @@ class TestModelSetup:
assert setup_with_gen.model is None
setup_with_gen.build_model()
assert isinstance(setup_with_gen.model, AbstractModelClass)
- expected = {"window_history_size", "window_lead_time", "channels", "dropout_rate", "regularizer", "initial_lr",
- "optimizer", "activation"}
+ expected = {"lr_decay", "model_name", "dropout_rate", "regularizer", "initial_lr", "optimizer", "activation",
+ "shape_inputs", "shape_outputs"}
assert expected <= self.current_scope_as_set(setup_with_gen)
- def test_set_channels(self, setup_with_gen_tiny):
- assert len(setup_with_gen_tiny.data_store.search_name("channels")) == 0
- setup_with_gen_tiny._set_channels()
- assert setup_with_gen_tiny.data_store.get("channels", setup_with_gen_tiny.scope) == 2
+ def test_set_shapes(self, setup_with_gen_tiny):
+ assert len(setup_with_gen_tiny.data_store.search_name("shape_inputs")) == 0
+ assert len(setup_with_gen_tiny.data_store.search_name("shape_outputs")) == 0
+ setup_with_gen_tiny._set_shapes()
+ assert setup_with_gen_tiny.data_store.get("shape_inputs", setup_with_gen_tiny.scope) == [(14, 1, 5), (10, 1, 2),
+ (1, 1, 2)]
+ assert setup_with_gen_tiny.data_store.get("shape_outputs", setup_with_gen_tiny.scope) == [(5,), (3,)]
def test_load_weights(self):
pass
@@ -109,3 +125,20 @@ class TestModelSetup:
def test_init(self):
pass
+
+
+class DummyData:
+
+ def __init__(self, number_of_samples=np.random.randint(100, 150)):
+ self.number_of_samples = number_of_samples
+
+ def get_X(self, upsampling=False, as_numpy=True):
+ X1 = np.random.randint(0, 10, size=(self.number_of_samples, 14, 1, 5)) # samples, window, variables
+ X2 = np.random.randint(21, 30, size=(self.number_of_samples, 10, 1, 2)) # samples, window, variables
+ X3 = np.random.randint(-5, 0, size=(self.number_of_samples, 1, 1, 2)) # samples, window, variables
+ return [X1, X2, X3]
+
+ def get_Y(self, upsampling=False, as_numpy=True):
+ Y1 = np.random.randint(0, 10, size=(self.number_of_samples, 5)) # samples, window
+ Y2 = np.random.randint(21, 30, size=(self.number_of_samples, 3)) # samples, window
+ return [Y1, Y2]
\ No newline at end of file
diff --git a/test/test_modules/test_pre_processing.py b/test/test_modules/test_pre_processing.py
index 0b439e9e..4bf23e6b 100644
--- a/test/test_modules/test_pre_processing.py
+++ b/test/test_modules/test_pre_processing.py
@@ -42,7 +42,7 @@ class TestPreProcessing:
caplog.set_level(logging.INFO)
with PreProcessing():
assert caplog.record_tuples[0] == ('root', 20, 'PreProcessing started')
- assert caplog.record_tuples[1] == ('root', 20, 'check valid stations started (all)')
+ assert caplog.record_tuples[1] == ('root', 20, 'check valid stations started (preprocessing)')
assert caplog.record_tuples[-1] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 5 '
r'station\(s\). Found 5/5 valid stations.'))
RunEnvironment().__del__()
--
GitLab