diff --git a/run.py b/run.py
index eaedb838135344df8a6b28339fa81660dfc85458..1579ae35f0d270dc0d2529cf1b6d36bc410e317a 100644
--- a/run.py
+++ b/run.py
@@ -4,7 +4,7 @@ __date__ = '2019-11-14'
import logging
import argparse
-from src.experiment_setup import ExperimentSetup
+from src.modules.experiment_setup import ExperimentSetup
from src.modules import run, PreProcessing, Training, PostProcessing
diff --git a/src/modules/__init__.py b/src/modules/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/src/experiment_setup.py b/src/modules/experiment_setup.py
similarity index 80%
rename from src/experiment_setup.py
rename to src/modules/experiment_setup.py
index ce43b38ef631c3dfbb176d4301c0a28349abe1f5..1e20871e92cf597368af1650cf6447bf10433981 100644
--- a/src/experiment_setup.py
+++ b/src/modules/experiment_setup.py
@@ -1,9 +1,14 @@
__author__ = "Lukas Leufen"
__date__ = '2019-11-15'
-from src import helpers, modules
+
import logging
import argparse
+from typing import Union, Dict, Any
+
+from src import helpers
+from src.modules.run_environment import RunEnvironment
+
DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004',
'DEBY020', 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073', 'DEBY039', 'DEBW038', 'DEBW081',
@@ -16,7 +21,7 @@ DEFAULT_VAR_ALL_DICT = {'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'max
'pblheight': 'maximum'}
-class ExperimentSetup(modules.run):
+class ExperimentSetup(RunEnvironment):
"""
params:
trainable: Train new model if true, otherwise try to load existing model
@@ -71,20 +76,32 @@ class ExperimentSetup(modules.run):
# use all stations on all data sets (train, val, test)
self._set_param("use_all_stations_on_all_data_sets", use_all_stations_on_all_data_sets, default=True)
- def _set_param(self, param, value, default=None, scope="general"):
+ def _set_param(self, param: str, value: Any, default: Any = None, scope: str = "general") -> None:
if value is None and default is not None:
value = default
self.data_store.put(param, value, scope)
- # setattr(self, param, value)
logging.debug(f"set experiment attribute: {param}({scope})={value}")
@staticmethod
- def _get_parser_args(args):
+ def _get_parser_args(args: Union[Dict, argparse.ArgumentParser]) -> Dict:
"""
Transform args to dict if given as argparse.Namespace
- :param args:
- :return:
+ :param args: either a dictionary or an argument parser instance
+ :return: dictionary with all arguments
"""
if isinstance(args, argparse.Namespace):
return args.__dict__
return args
+
+
+if __name__ == "__main__":
+
+ formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
+ logging.basicConfig(format=formatter, level=logging.DEBUG)
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None,
+ help="set experiment date as string")
+ parser_args = parser.parse_args()
+ with RunEnvironment():
+ setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
diff --git a/src/modules/modules.py b/src/modules/modules.py
new file mode 100644
index 0000000000000000000000000000000000000000..8532e1d812a5de7a3b47423d9f9bb3c9bcd43abc
--- /dev/null
+++ b/src/modules/modules.py
@@ -0,0 +1,33 @@
+import logging
+# from src.experiment_setup import ExperimentSetup
+import argparse
+
+from src.modules.run_environment import RunEnvironment
+
+
+class Training(RunEnvironment):
+
+ def __init__(self, setup):
+ super().__init__()
+ self.setup = setup
+
+
+class PostProcessing(RunEnvironment):
+
+ def __init__(self, setup):
+ super().__init__()
+ self.setup = setup
+
+
+if __name__ == "__main__":
+
+ formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
+ logging.basicConfig(format=formatter, level=logging.DEBUG)
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None,
+ help="set experiment date as string")
+ parser_args = parser.parse_args()
+ # with run():
+ # setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
+ # PreProcessing(setup)
diff --git a/src/modules.py b/src/modules/pre_processing.py
similarity index 74%
rename from src/modules.py
rename to src/modules/pre_processing.py
index 8e1b821e5851f8b00284949f454ef809fb1ad78d..141d9abec1757dce57f808c133dd653e8c249287 100644
--- a/src/modules.py
+++ b/src/modules/pre_processing.py
@@ -1,51 +1,12 @@
-from src.helpers import TimeTracking
import logging
-import time
-from src.data_generator import DataGenerator
-# from src.experiment_setup import ExperimentSetup
-from src.datastore import DataStoreByScope as DataStoreObject
-import argparse
-from typing import Dict, List, Any, Tuple
-
-
-class run(object):
- """
- basic run class to measure execution time. Either call this class calling it by 'with' or delete the class instance
- after finishing the measurement. The duration result is logged.
- """
-
- del_by_exit = False
- data_store = DataStoreObject()
-
- def __init__(self):
- """
- Starts time tracking automatically and logs as info.
- """
- self.time = TimeTracking()
- logging.info(f"{self.__class__.__name__} started")
-
- def __del__(self):
- """
- This is the class finalizer. The code is not executed if already called by exit method to prevent duplicated
- logging (__exit__ is always executed before __del__) it this class was used in a with statement.
- """
- if not self.del_by_exit:
- self.time.stop()
- logging.info(f"{self.__class__.__name__} finished after {self.time}")
- self.del_by_exit = True
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.__del__()
+from typing import Any, Tuple, Dict, List
- @staticmethod
- def do_stuff(length=2):
- time.sleep(length)
+from src.data_generator import DataGenerator
+from src.helpers import TimeTracking
+from src.modules.run_environment import RunEnvironment
-class PreProcessing(run):
+class PreProcessing(RunEnvironment):
"""
Pre-process your data by using this class. It includes time tracking and uses the experiment setup to look for data
@@ -166,32 +127,4 @@ class PreProcessing(run):
except AttributeError:
continue
logging.info(f"run for {t_outer} to check {len(all_stations)} station(s)")
- return valid_stations
-
-
-class Training(run):
-
- def __init__(self, setup):
- super().__init__()
- self.setup = setup
-
-
-class PostProcessing(run):
-
- def __init__(self, setup):
- super().__init__()
- self.setup = setup
-
-
-if __name__ == "__main__":
-
- formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
- logging.basicConfig(format=formatter, level=logging.DEBUG)
-
- parser = argparse.ArgumentParser()
- parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None,
- help="set experiment date as string")
- parser_args = parser.parse_args()
- # with run():
- # setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
- # PreProcessing(setup)
+ return valid_stations
\ No newline at end of file
diff --git a/src/modules/run_environment.py b/src/modules/run_environment.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0aa77d50fc4fbc10b3b9e4debfe2ae5173d2a22
--- /dev/null
+++ b/src/modules/run_environment.py
@@ -0,0 +1,45 @@
+__author__ = "Lukas Leufen"
+__date__ = '2019-11-25'
+
+import logging
+import time
+
+from src.datastore import DataStoreByScope as DataStoreObject
+from src.helpers import TimeTracking
+
+
+class RunEnvironment(object):
+ """
+ basic run class to measure execution time. Either call this class calling it by 'with' or delete the class instance
+ after finishing the measurement. The duration result is logged.
+ """
+
+ del_by_exit = False
+ data_store = DataStoreObject()
+
+ def __init__(self):
+ """
+ Starts time tracking automatically and logs as info.
+ """
+ self.time = TimeTracking()
+ logging.info(f"{self.__class__.__name__} started")
+
+ def __del__(self):
+ """
+ This is the class finalizer. The code is not executed if already called by exit method to prevent duplicated
+ logging (__exit__ is always executed before __del__) it this class was used in a with statement.
+ """
+ if not self.del_by_exit:
+ self.time.stop()
+ logging.info(f"{self.__class__.__name__} finished after {self.time}")
+ self.del_by_exit = True
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.__del__()
+
+ @staticmethod
+ def do_stuff(length=2):
+ time.sleep(length)
diff --git a/test/test_modules.py b/test/test_modules.py
index 3211a8e855e8b342b42b1b86793df44fd061c179..8437c9129737ff91d914af42653b0467d635123b 100644
--- a/test/test_modules.py
+++ b/test/test_modules.py
@@ -1,9 +1,7 @@
-import pytest
import logging
from src.modules import run, PreProcessing
from src.helpers import TimeTracking
-import src.helpers
-from src.experiment_setup import ExperimentSetup
+from src.modules.experiment_setup import ExperimentSetup
from src.data_generator import DataGenerator
import re
import mock