From bf2208c5b5b4ba2dbb16d6f8e5444e6e146df338 Mon Sep 17 00:00:00 2001
From: lukas leufen <l.leufen@fz-juelich.de>
Date: Wed, 11 Dec 2019 09:54:39 +0100
Subject: [PATCH] new directory model_modules to collect all model methods and
classes
---
CI/run_pytest.sh | 2 +-
run.py | 12 ++++++------
src/data_handling/data_distributor.py | 4 ++++
src/datastore.py | 6 +++---
src/helpers.py | 2 +-
src/{modules => model_modules}/__init__.py | 0
src/{ => model_modules}/flatten.py | 2 +-
src/{ => model_modules}/inception_model.py | 0
src/run_modules/__init__.py | 0
src/{modules => run_modules}/experiment_setup.py | 4 ++--
src/{modules => run_modules}/model_setup.py | 10 +++++-----
src/{modules => run_modules}/modules.py | 6 +++---
src/run_modules/post_processing.py | 15 +++++++++++++++
src/{modules => run_modules}/pre_processing.py | 4 ++--
src/{modules => run_modules}/run_environment.py | 0
src/{modules => run_modules}/training.py | 3 +--
.../test_inception_model.py | 3 +--
test/test_modules/test_experiment_setup.py | 2 +-
test/test_modules/test_model_setup.py | 4 ++--
test/test_modules/test_post_processing.py | 10 ++++++++++
test/test_modules/test_pre_processing.py | 6 +++---
test/test_modules/test_run_environment.py | 2 +-
test/test_modules/test_training.py | 8 ++++----
23 files changed, 66 insertions(+), 39 deletions(-)
rename src/{modules => model_modules}/__init__.py (100%)
rename src/{ => model_modules}/flatten.py (96%)
rename src/{ => model_modules}/inception_model.py (100%)
create mode 100644 src/run_modules/__init__.py
rename src/{modules => run_modules}/experiment_setup.py (98%)
rename src/{modules => run_modules}/model_setup.py (97%)
rename src/{modules => run_modules}/modules.py (82%)
create mode 100644 src/run_modules/post_processing.py
rename src/{modules => run_modules}/pre_processing.py (98%)
rename src/{modules => run_modules}/run_environment.py (100%)
rename src/{modules => run_modules}/training.py (98%)
rename test/{ => test_model_modules}/test_inception_model.py (99%)
create mode 100644 test/test_modules/test_post_processing.py
diff --git a/CI/run_pytest.sh b/CI/run_pytest.sh
index 5547b7ab..a7d883dc 100644
--- a/CI/run_pytest.sh
+++ b/CI/run_pytest.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# run pytest for all modules
+# run pytest for all run_modules
python3 -m pytest --html=report.html --self-contained-html test/ | tee test_results.out
IS_FAILED=$?
diff --git a/run.py b/run.py
index e45b2dd6..18189f95 100644
--- a/run.py
+++ b/run.py
@@ -5,12 +5,12 @@ __date__ = '2019-11-14'
import logging
import argparse
-from src.modules.experiment_setup import ExperimentSetup
-from src.modules.run_environment import RunEnvironment
-from src.modules.pre_processing import PreProcessing
-from src.modules.model_setup import ModelSetup
-from src.modules.training import Training
-from src.modules.modules import PostProcessing
+from src.run_modules.experiment_setup import ExperimentSetup
+from src.run_modules.run_environment import RunEnvironment
+from src.run_modules.pre_processing import PreProcessing
+from src.run_modules.model_setup import ModelSetup
+from src.run_modules.training import Training
+from src.run_modules.modules import PostProcessing
def main(parser_args):
diff --git a/src/data_handling/data_distributor.py b/src/data_handling/data_distributor.py
index 77f83536..74df5f6a 100644
--- a/src/data_handling/data_distributor.py
+++ b/src/data_handling/data_distributor.py
@@ -1,4 +1,8 @@
from __future__ import generator_stop
+
+__author__ = "Lukas Leufen, Felix Kleinert"
+__date__ = '2019-12-05'
+
import math
import keras
diff --git a/src/datastore.py b/src/datastore.py
index 5f0df675..923603f0 100644
--- a/src/datastore.py
+++ b/src/datastore.py
@@ -30,7 +30,7 @@ class EmptyScope(Exception):
class AbstractDataStore(ABC):
"""
- Data store for all settings for the experiment workflow to save experiment parameters for the proceeding modules
+ Data store for all settings for the experiment workflow to save experiment parameters for the proceeding run_modules
and predefine parameters loaded during the experiment setup phase. The data store is hierarchically structured, so
that global settings can be overwritten by local adjustments.
"""
@@ -102,7 +102,7 @@ class AbstractDataStore(ABC):
class DataStoreByVariable(AbstractDataStore):
"""
- Data store for all settings for the experiment workflow to save experiment parameters for the proceeding modules
+ Data store for all settings for the experiment workflow to save experiment parameters for the proceeding run_modules
and predefine parameters loaded during the experiment setup phase. The data store is hierarchically structured, so
that global settings can be overwritten by local adjustments.
@@ -226,7 +226,7 @@ class DataStoreByVariable(AbstractDataStore):
class DataStoreByScope(AbstractDataStore):
"""
- Data store for all settings for the experiment workflow to save experiment parameters for the proceeding modules
+ Data store for all settings for the experiment workflow to save experiment parameters for the proceeding run_modules
and predefine parameters loaded during the experiment setup phase. The data store is hierarchically structured, so
that global settings can be overwritten by local adjustments.
diff --git a/src/helpers.py b/src/helpers.py
index 2ef77689..40a3f976 100644
--- a/src/helpers.py
+++ b/src/helpers.py
@@ -1,6 +1,6 @@
import re
-__author__ = 'Lukas Leufen'
+__author__ = 'Lukas Leufen, Felix Kleinert'
__date__ = '2019-10-21'
diff --git a/src/modules/__init__.py b/src/model_modules/__init__.py
similarity index 100%
rename from src/modules/__init__.py
rename to src/model_modules/__init__.py
diff --git a/src/flatten.py b/src/model_modules/flatten.py
similarity index 96%
rename from src/flatten.py
rename to src/model_modules/flatten.py
index 1166cf32..707b0d44 100644
--- a/src/flatten.py
+++ b/src/model_modules/flatten.py
@@ -1,4 +1,4 @@
-__author__ = "Lukas Leufen"
+__author__ = "Felix Kleinert, Lukas Leufen"
__date__ = '2019-12-02'
import keras
diff --git a/src/inception_model.py b/src/model_modules/inception_model.py
similarity index 100%
rename from src/inception_model.py
rename to src/model_modules/inception_model.py
diff --git a/src/run_modules/__init__.py b/src/run_modules/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/modules/experiment_setup.py b/src/run_modules/experiment_setup.py
similarity index 98%
rename from src/modules/experiment_setup.py
rename to src/run_modules/experiment_setup.py
index 472173c7..f502e81b 100644
--- a/src/modules/experiment_setup.py
+++ b/src/run_modules/experiment_setup.py
@@ -1,4 +1,4 @@
-__author__ = "Lukas Leufen"
+__author__ = "Lukas Leufen, Felix Kleinert"
__date__ = '2019-11-15'
@@ -7,7 +7,7 @@ import argparse
from typing import Union, Dict, Any
from src import helpers
-from src.modules.run_environment import RunEnvironment
+from src.run_modules.run_environment import RunEnvironment
DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004',
diff --git a/src/modules/model_setup.py b/src/run_modules/model_setup.py
similarity index 97%
rename from src/modules/model_setup.py
rename to src/run_modules/model_setup.py
index f6c25aff..43625bdf 100644
--- a/src/modules/model_setup.py
+++ b/src/run_modules/model_setup.py
@@ -1,4 +1,4 @@
-__author__ = "Lukas Leufen"
+__author__ = "Lukas Leufen, Felix Kleinert"
__date__ = '2019-12-02'
@@ -6,15 +6,15 @@ import keras
from keras import losses
from keras.callbacks import ModelCheckpoint
from keras.regularizers import l2
-from keras.optimizers import Adam, SGD
+from keras.optimizers import SGD
import tensorflow as tf
import logging
import os
-from src.modules.run_environment import RunEnvironment
+from src.run_modules.run_environment import RunEnvironment
from src.helpers import l_p_loss, LearningRateDecay
-from src.inception_model import InceptionModelBase
-from src.flatten import flatten_tail
+from src.model_modules.inception_model import InceptionModelBase
+from src.model_modules.flatten import flatten_tail
class ModelSetup(RunEnvironment):
diff --git a/src/modules/modules.py b/src/run_modules/modules.py
similarity index 82%
rename from src/modules/modules.py
rename to src/run_modules/modules.py
index 888c7e06..5f0f12c1 100644
--- a/src/modules/modules.py
+++ b/src/run_modules/modules.py
@@ -1,9 +1,9 @@
import logging
import argparse
-from src.modules.run_environment import RunEnvironment
-from src.modules.experiment_setup import ExperimentSetup
-from src.modules.pre_processing import PreProcessing
+from src.run_modules.run_environment import RunEnvironment
+from src.run_modules.experiment_setup import ExperimentSetup
+from src.run_modules.pre_processing import PreProcessing
class Training(RunEnvironment):
diff --git a/src/run_modules/post_processing.py b/src/run_modules/post_processing.py
new file mode 100644
index 00000000..c76ef1cc
--- /dev/null
+++ b/src/run_modules/post_processing.py
@@ -0,0 +1,15 @@
+__author__ = "Lukas Leufen, Felix Kleinert"
+__date__ = '2019-12-11'
+
+
+from src.run_modules.run_environment import RunEnvironment
+
+
+class PostProcessing(RunEnvironment):
+
+ def __init__(self):
+ super().__init__()
+ self._run()
+
+ def _run(self):
+ pass
diff --git a/src/modules/pre_processing.py b/src/run_modules/pre_processing.py
similarity index 98%
rename from src/modules/pre_processing.py
rename to src/run_modules/pre_processing.py
index 8fad9d1b..8e11877f 100644
--- a/src/modules/pre_processing.py
+++ b/src/run_modules/pre_processing.py
@@ -1,4 +1,4 @@
-__author__ = "Lukas Leufen"
+__author__ = "Lukas Leufen, Felix Kleinert"
__date__ = '2019-11-25'
@@ -7,7 +7,7 @@ from typing import Tuple, Dict, List
from src.data_handling.data_generator import DataGenerator
from src.helpers import TimeTracking
-from src.modules.run_environment import RunEnvironment
+from src.run_modules.run_environment import RunEnvironment
from src.join import EmptyQueryResult
diff --git a/src/modules/run_environment.py b/src/run_modules/run_environment.py
similarity index 100%
rename from src/modules/run_environment.py
rename to src/run_modules/run_environment.py
diff --git a/src/modules/training.py b/src/run_modules/training.py
similarity index 98%
rename from src/modules/training.py
rename to src/run_modules/training.py
index 87dcf35e..5dab3da2 100644
--- a/src/modules/training.py
+++ b/src/run_modules/training.py
@@ -1,4 +1,3 @@
-
__author__ = "Lukas Leufen, Felix Kleinert"
__date__ = '2019-12-05'
@@ -7,7 +6,7 @@ import os
import json
import keras
-from src.modules.run_environment import RunEnvironment
+from src.run_modules.run_environment import RunEnvironment
from src.data_handling.data_distributor import Distributor
diff --git a/test/test_inception_model.py b/test/test_model_modules/test_inception_model.py
similarity index 99%
rename from test/test_inception_model.py
rename to test/test_model_modules/test_inception_model.py
index ad180595..82b754c2 100644
--- a/test/test_inception_model.py
+++ b/test/test_model_modules/test_inception_model.py
@@ -1,7 +1,6 @@
import pytest
-from src.inception_model import InceptionModelBase
+from src.model_modules.inception_model import InceptionModelBase
import keras
-import tensorflow as tf
class TestInceptionModelBase:
diff --git a/test/test_modules/test_experiment_setup.py b/test/test_modules/test_experiment_setup.py
index bfff606e..4a2ba9f9 100644
--- a/test/test_modules/test_experiment_setup.py
+++ b/test/test_modules/test_experiment_setup.py
@@ -3,7 +3,7 @@ import logging
import argparse
import os
-from src.modules.experiment_setup import ExperimentSetup
+from src.run_modules.experiment_setup import ExperimentSetup
from src.helpers import TimeTracking, prepare_host
from src.datastore import NameNotFoundInScope, NameNotFoundInDataStore
diff --git a/test/test_modules/test_model_setup.py b/test/test_modules/test_model_setup.py
index 85cb24e3..d1cfe489 100644
--- a/test/test_modules/test_model_setup.py
+++ b/test/test_modules/test_model_setup.py
@@ -2,8 +2,8 @@ import pytest
import os
import keras
-from src.modules.model_setup import ModelSetup
-from src.modules.run_environment import RunEnvironment
+from src.run_modules.model_setup import ModelSetup
+from src.run_modules.run_environment import RunEnvironment
from src.data_handling.data_generator import DataGenerator
diff --git a/test/test_modules/test_post_processing.py b/test/test_modules/test_post_processing.py
new file mode 100644
index 00000000..0eb7f741
--- /dev/null
+++ b/test/test_modules/test_post_processing.py
@@ -0,0 +1,10 @@
+import keras
+
+from src.run_modules.post_processing import PostProcessing
+
+
+class TestPostProcessing:
+
+ def test_init(self):
+ post = PostProcessing()
+ assert isinstance(post.model, keras.Model)
diff --git a/test/test_modules/test_pre_processing.py b/test/test_modules/test_pre_processing.py
index c884b146..14dd425a 100644
--- a/test/test_modules/test_pre_processing.py
+++ b/test/test_modules/test_pre_processing.py
@@ -2,11 +2,11 @@ import logging
import pytest
from src.helpers import PyTestRegex
-from src.modules.experiment_setup import ExperimentSetup
-from src.modules.pre_processing import PreProcessing, DEFAULT_ARGS_LIST, DEFAULT_KWARGS_LIST
+from src.run_modules.experiment_setup import ExperimentSetup
+from src.run_modules.pre_processing import PreProcessing, DEFAULT_ARGS_LIST, DEFAULT_KWARGS_LIST
from src.data_handling.data_generator import DataGenerator
from src.datastore import NameNotFoundInScope
-from src.modules.run_environment import RunEnvironment
+from src.run_modules.run_environment import RunEnvironment
class TestPreProcessing:
diff --git a/test/test_modules/test_run_environment.py b/test/test_modules/test_run_environment.py
index 1eeaa02c..d82675b5 100644
--- a/test/test_modules/test_run_environment.py
+++ b/test/test_modules/test_run_environment.py
@@ -1,7 +1,7 @@
import logging
from src.helpers import TimeTracking, PyTestRegex
-from src.modules.run_environment import RunEnvironment
+from src.run_modules.run_environment import RunEnvironment
class TestRunEnvironment:
diff --git a/test/test_modules/test_training.py b/test/test_modules/test_training.py
index c91cb691..01cc0804 100644
--- a/test/test_modules/test_training.py
+++ b/test/test_modules/test_training.py
@@ -7,10 +7,10 @@ import json
import shutil
import logging
-from src.inception_model import InceptionModelBase
-from src.flatten import flatten_tail
-from src.modules.training import Training
-from src.modules.run_environment import RunEnvironment
+from src.model_modules.inception_model import InceptionModelBase
+from src.model_modules.flatten import flatten_tail
+from src.run_modules.training import Training
+from src.run_modules.run_environment import RunEnvironment
from src.data_handling.data_distributor import Distributor
from src.data_handling.data_generator import DataGenerator
from src.helpers import LearningRateDecay, PyTestRegex
--
GitLab