From 2a101b8021c39072971fd90c97c3d6c95be3cf4f Mon Sep 17 00:00:00 2001
From: lukas leufen <l.leufen@fz-juelich.de>
Date: Mon, 25 Nov 2019 14:46:41 +0100
Subject: [PATCH] introduced module "modules" to collect all working steps
(like experiment setup, pre-processing) in separate folder
some paths are still not updated and will cause errors (WIP)
---
run.py | 2 +-
src/modules/__init__.py | 0
src/{ => modules}/experiment_setup.py | 31 ++++++--
src/modules/modules.py | 33 ++++++++
src/{modules.py => modules/pre_processing.py} | 79 ++-----------------
src/modules/run_environment.py | 45 +++++++++++
test/test_modules.py | 4 +-
7 files changed, 110 insertions(+), 84 deletions(-)
create mode 100644 src/modules/__init__.py
rename src/{ => modules}/experiment_setup.py (80%)
create mode 100644 src/modules/modules.py
rename src/{modules.py => modules/pre_processing.py} (74%)
create mode 100644 src/modules/run_environment.py
diff --git a/run.py b/run.py
index eaedb838..1579ae35 100644
--- a/run.py
+++ b/run.py
@@ -4,7 +4,7 @@ __date__ = '2019-11-14'
import logging
import argparse
-from src.experiment_setup import ExperimentSetup
+from src.modules.experiment_setup import ExperimentSetup
from src.modules import run, PreProcessing, Training, PostProcessing
diff --git a/src/modules/__init__.py b/src/modules/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/experiment_setup.py b/src/modules/experiment_setup.py
similarity index 80%
rename from src/experiment_setup.py
rename to src/modules/experiment_setup.py
index ce43b38e..1e20871e 100644
--- a/src/experiment_setup.py
+++ b/src/modules/experiment_setup.py
@@ -1,9 +1,14 @@
__author__ = "Lukas Leufen"
__date__ = '2019-11-15'
-from src import helpers, modules
+
import logging
import argparse
+from typing import Union, Dict, Any
+
+from src import helpers
+from src.modules.run_environment import RunEnvironment
+
DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004',
'DEBY020', 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073', 'DEBY039', 'DEBW038', 'DEBW081',
@@ -16,7 +21,7 @@ DEFAULT_VAR_ALL_DICT = {'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'max
'pblheight': 'maximum'}
-class ExperimentSetup(modules.run):
+class ExperimentSetup(RunEnvironment):
"""
params:
trainable: Train new model if true, otherwise try to load existing model
@@ -71,20 +76,32 @@ class ExperimentSetup(modules.run):
# use all stations on all data sets (train, val, test)
self._set_param("use_all_stations_on_all_data_sets", use_all_stations_on_all_data_sets, default=True)
- def _set_param(self, param, value, default=None, scope="general"):
+ def _set_param(self, param: str, value: Any, default: Any = None, scope: str = "general") -> None:
if value is None and default is not None:
value = default
self.data_store.put(param, value, scope)
- # setattr(self, param, value)
logging.debug(f"set experiment attribute: {param}({scope})={value}")
@staticmethod
- def _get_parser_args(args):
+ def _get_parser_args(args: Union[Dict, argparse.ArgumentParser]) -> Dict:
"""
Transform args to dict if given as argparse.Namespace
- :param args:
- :return:
+ :param args: either a dictionary or an argument parser instance
+ :return: dictionary with all arguments
"""
if isinstance(args, argparse.Namespace):
return args.__dict__
return args
+
+
+if __name__ == "__main__":
+
+ formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
+ logging.basicConfig(format=formatter, level=logging.DEBUG)
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None,
+ help="set experiment date as string")
+ parser_args = parser.parse_args()
+ with RunEnvironment():
+ setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
diff --git a/src/modules/modules.py b/src/modules/modules.py
new file mode 100644
index 00000000..8532e1d8
--- /dev/null
+++ b/src/modules/modules.py
@@ -0,0 +1,33 @@
+import logging
+# from src.experiment_setup import ExperimentSetup
+import argparse
+
+from src.modules.run_environment import RunEnvironment
+
+
+class Training(RunEnvironment):
+
+ def __init__(self, setup):
+ super().__init__()
+ self.setup = setup
+
+
+class PostProcessing(RunEnvironment):
+
+ def __init__(self, setup):
+ super().__init__()
+ self.setup = setup
+
+
+if __name__ == "__main__":
+
+ formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
+ logging.basicConfig(format=formatter, level=logging.DEBUG)
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None,
+ help="set experiment date as string")
+ parser_args = parser.parse_args()
+ # with run():
+ # setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
+ # PreProcessing(setup)
diff --git a/src/modules.py b/src/modules/pre_processing.py
similarity index 74%
rename from src/modules.py
rename to src/modules/pre_processing.py
index 8e1b821e..141d9abe 100644
--- a/src/modules.py
+++ b/src/modules/pre_processing.py
@@ -1,51 +1,12 @@
-from src.helpers import TimeTracking
import logging
-import time
-from src.data_generator import DataGenerator
-# from src.experiment_setup import ExperimentSetup
-from src.datastore import DataStoreByScope as DataStoreObject
-import argparse
-from typing import Dict, List, Any, Tuple
-
-
-class run(object):
- """
- basic run class to measure execution time. Either call this class calling it by 'with' or delete the class instance
- after finishing the measurement. The duration result is logged.
- """
-
- del_by_exit = False
- data_store = DataStoreObject()
-
- def __init__(self):
- """
- Starts time tracking automatically and logs as info.
- """
- self.time = TimeTracking()
- logging.info(f"{self.__class__.__name__} started")
-
- def __del__(self):
- """
- This is the class finalizer. The code is not executed if already called by exit method to prevent duplicated
- logging (__exit__ is always executed before __del__) it this class was used in a with statement.
- """
- if not self.del_by_exit:
- self.time.stop()
- logging.info(f"{self.__class__.__name__} finished after {self.time}")
- self.del_by_exit = True
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- self.__del__()
+from typing import Any, Tuple, Dict, List
- @staticmethod
- def do_stuff(length=2):
- time.sleep(length)
+from src.data_generator import DataGenerator
+from src.helpers import TimeTracking
+from src.modules.run_environment import RunEnvironment
-class PreProcessing(run):
+class PreProcessing(RunEnvironment):
"""
Pre-process your data by using this class. It includes time tracking and uses the experiment setup to look for data
@@ -166,32 +127,4 @@ class PreProcessing(run):
except AttributeError:
continue
logging.info(f"run for {t_outer} to check {len(all_stations)} station(s)")
- return valid_stations
-
-
-class Training(run):
-
- def __init__(self, setup):
- super().__init__()
- self.setup = setup
-
-
-class PostProcessing(run):
-
- def __init__(self, setup):
- super().__init__()
- self.setup = setup
-
-
-if __name__ == "__main__":
-
- formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
- logging.basicConfig(format=formatter, level=logging.DEBUG)
-
- parser = argparse.ArgumentParser()
- parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None,
- help="set experiment date as string")
- parser_args = parser.parse_args()
- # with run():
- # setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
- # PreProcessing(setup)
+ return valid_stations
\ No newline at end of file
diff --git a/src/modules/run_environment.py b/src/modules/run_environment.py
new file mode 100644
index 00000000..b0aa77d5
--- /dev/null
+++ b/src/modules/run_environment.py
@@ -0,0 +1,45 @@
+__author__ = "Lukas Leufen"
+__date__ = '2019-11-25'
+
+import logging
+import time
+
+from src.datastore import DataStoreByScope as DataStoreObject
+from src.helpers import TimeTracking
+
+
+class RunEnvironment(object):
+ """
+ basic run class to measure execution time. Either call this class calling it by 'with' or delete the class instance
+ after finishing the measurement. The duration result is logged.
+ """
+
+ del_by_exit = False
+ data_store = DataStoreObject()
+
+ def __init__(self):
+ """
+ Starts time tracking automatically and logs as info.
+ """
+ self.time = TimeTracking()
+ logging.info(f"{self.__class__.__name__} started")
+
+ def __del__(self):
+ """
+ This is the class finalizer. The code is not executed if already called by exit method to prevent duplicated
+ logging (__exit__ is always executed before __del__) it this class was used in a with statement.
+ """
+ if not self.del_by_exit:
+ self.time.stop()
+ logging.info(f"{self.__class__.__name__} finished after {self.time}")
+ self.del_by_exit = True
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ self.__del__()
+
+ @staticmethod
+ def do_stuff(length=2):
+ time.sleep(length)
diff --git a/test/test_modules.py b/test/test_modules.py
index 3211a8e8..8437c912 100644
--- a/test/test_modules.py
+++ b/test/test_modules.py
@@ -1,9 +1,7 @@
-import pytest
import logging
from src.modules import run, PreProcessing
from src.helpers import TimeTracking
-import src.helpers
-from src.experiment_setup import ExperimentSetup
+from src.modules.experiment_setup import ExperimentSetup
from src.data_generator import DataGenerator
import re
import mock
--
GitLab