split run.py into experiment_setup.py and modules.py (includes preprocessing,...

split run.py into experiment_setup.py and modules.py (includes preprocessing, training, postprocessing)

run.py

@@ -3,133 +3,15 @@ __date__ = '2019-11-14'
 
 
 import logging
-from src.helpers import TimeTracking
-from src import helpers
 import argparse
-import time
+from src.experiment_setup import ExperimentSetup
+from src.modules import run, PreProcessing, Training, PostProcessing
 
 
 formatter = "%(asctime)s - %(levelname)s: %(message)s  [%(filename)s:%(funcName)s:%(lineno)s]"
 logging.basicConfig(level=logging.INFO, format=formatter)
 
 
-class run(object):
-    """
-    basic run class to measure execution time. Either call this class calling it by 'with' or delete the class instance
-    after finishing the measurement. The duration result is logged.
-    """
-
-    def __init__(self):
-        self.time = TimeTracking()
-        logging.info(f"{self.__class__.__name__} started")
-
-    def __del__(self):
-        self.time.stop()
-        logging.info(f"{self.__class__.__name__} finished after {self.time}")
-
-    def __enter__(self):
-        pass
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
-
-    def do_stuff(self):
-        time.sleep(2)
-
-
-class ExperimentSetup:
-    """
-    params:
-    trainable: Train new model if true, otherwise try to load existing model
-    """
-
-    def __init__(self, **kwargs):
-        self.data_path = None
-        self.experiment_path = None
-        self.experiment_name = None
-        self.trainable = None
-        self.fraction_of_train = None
-        self.use_all_stations_on_all_data_sets = None
-        self.network = None
-        self.var_all_dict = None
-        self.all_stations = None
-        self.variables = None
-        self.dimensions = None
-        self.dim = None
-        self.target_dim = None
-        self.target_var = None
-        self.setup_experiment(**kwargs)
-
-    def _set_param(self, param, value, default=None):
-        if default is not None:
-            value = value.get(param, default)
-        setattr(self, param, value)
-        logging.info(f"set experiment attribute: {param}={value}")
-
-    def setup_experiment(self, **kwargs):
-
-        # set data path of this experiment
-        self._set_param("data_path", helpers.prepare_host())
-
-        # set experiment name
-        exp_date = args.experiment_date
-        exp_name, exp_path = helpers.set_experiment_name(experiment_date=exp_date)
-        self._set_param("experiment_name", exp_name)
-        self._set_param("experiment_path", exp_path)
-        helpers.check_path_and_create(self.experiment_path)
-
-        # set if model is trainable
-        self._set_param("trainable", kwargs, default=True)
-
-        # set fraction of train
-        self._set_param("fraction_of_train", kwargs, default=0.8)
-
-        # use all stations on all data sets (train, val, test)
-        self._set_param("use_all_stations_on_all_data_sets", kwargs, default=True)
-        self._set_param("network", kwargs, default="AIRBASE")
-        self._set_param("var_all_dict", kwargs, default={'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum',
-                                                         'u': 'average_values', 'v': 'average_values', 'no': 'dma8eu',
-                                                         'no2': 'dma8eu', 'cloudcover': 'average_values',
-                                                         'pblheight': 'maximum'})
-        self._set_param("all_stations", kwargs, default=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087',
-                                                         'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004', 'DEBY020',
-                                                         'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073',
-                                                         'DEBY039', 'DEBW038', 'DEBW081', 'DEBY075', 'DEBW040',
-                                                         'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', 'DEBW042',
-                                                         'DEBW039', 'DEBY001', 'DEBY113', 'DEBY089', 'DEBW024',
-                                                         'DEBW004', 'DEBY037', 'DEBW056', 'DEBW029', 'DEBY068',
-                                                         'DEBW010', 'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084',
-                                                         'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063',
-                                                         'DEBY005', 'DEBW046', 'DEBW103', 'DEBW052', 'DEBW034',
-                                                         'DEBY088', ])
-        self._set_param("variables", kwargs, default=list(self.var_all_dict.keys()))
-        self._set_param("dimensions", kwargs, default={'new_index': ['datetime', 'Stations']})
-        self._set_param("dim", kwargs, default='datetime')
-        self._set_param("target_dim", kwargs, default='variables')
-        self._set_param("target_var", kwargs, default="o3")
-
-
-class PreProcessing(run):
-
-    def __init__(self, setup):
-        super().__init__()
-        self.setup = setup
-
-
-class Training(run):
-
-    def __init__(self, setup):
-        super().__init__()
-        self.setup = setup
-
-
-class PostProcessing(run):
-
-    def __init__(self, setup):
-        super().__init__()
-        self.setup = setup
-
-
 if __name__ == "__main__":
 
     parser = argparse.ArgumentParser()
@@ -138,7 +20,7 @@ if __name__ == "__main__":
     args = parser.parse_args()
 
     with run():
-        exp_setup = ExperimentSetup(trainable=True)
+        exp_setup = ExperimentSetup(args, trainable=True)
 
         PreProcessing(exp_setup)
 

src/experiment_setup.py

+__author__ = "Lukas Leufen"
+__date__ = '2019-11-15'
+
+
+from src import helpers
+import logging
+
+
+class ExperimentSetup:
+    """
+    params:
+    trainable: Train new model if true, otherwise try to load existing model
+    """
+
+    def __init__(self, parser_args, **kwargs):
+        self.args = parser_args
+        self.data_path = None
+        self.experiment_path = None
+        self.experiment_name = None
+        self.trainable = None
+        self.fraction_of_train = None
+        self.use_all_stations_on_all_data_sets = None
+        self.network = None
+        self.var_all_dict = None
+        self.all_stations = None
+        self.variables = None
+        self.dimensions = None
+        self.dim = None
+        self.target_dim = None
+        self.target_var = None
+        self.setup_experiment(**kwargs)
+
+    def _set_param(self, param, value, default=None):
+        if default is not None:
+            value = value.get(param, default)
+        setattr(self, param, value)
+        logging.debug(f"set experiment attribute: {param}={value}")
+
+    def setup_experiment(self, **kwargs):
+
+        # set data path of this experiment
+        self._set_param("data_path", helpers.prepare_host())
+
+        # set experiment name
+        exp_date = self.args.experiment_date
+        exp_name, exp_path = helpers.set_experiment_name(experiment_date=exp_date)
+        self._set_param("experiment_name", exp_name)
+        self._set_param("experiment_path", exp_path)
+        helpers.check_path_and_create(self.experiment_path)
+
+        # set if model is trainable
+        self._set_param("trainable", kwargs, default=True)
+
+        # set fraction of train
+        self._set_param("fraction_of_train", kwargs, default=0.8)
+
+        # use all stations on all data sets (train, val, test)
+        self._set_param("use_all_stations_on_all_data_sets", kwargs, default=True)
+        self._set_param("network", kwargs, default="AIRBASE")
+        self._set_param("var_all_dict", kwargs, default={'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum',
+                                                         'u': 'average_values', 'v': 'average_values', 'no': 'dma8eu',
+                                                         'no2': 'dma8eu', 'cloudcover': 'average_values',
+                                                         'pblheight': 'maximum'})
+        self._set_param("all_stations", kwargs, default=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087',
+                                                         'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004', 'DEBY020',
+                                                         'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073',
+                                                         'DEBY039', 'DEBW038', 'DEBW081', 'DEBY075', 'DEBW040',
+                                                         'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', 'DEBW042',
+                                                         'DEBW039', 'DEBY001', 'DEBY113', 'DEBY089', 'DEBW024',
+                                                         'DEBW004', 'DEBY037', 'DEBW056', 'DEBW029', 'DEBY068',
+                                                         'DEBW010', 'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084',
+                                                         'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063',
+                                                         'DEBY005', 'DEBW046', 'DEBW103', 'DEBW052', 'DEBW034',
+                                                         'DEBY088', ])
+        self._set_param("variables", kwargs, default=list(self.var_all_dict.keys()))
+        self._set_param("dimensions", kwargs, default={'new_index': ['datetime', 'Stations']})
+        self._set_param("dim", kwargs, default='datetime')
+        self._set_param("target_dim", kwargs, default='variables')
+        self._set_param("target_var", kwargs, default="o3")

src/modules.py

+from src.helpers import TimeTracking
+import logging
+import time
+
+
+class run(object):
+    """
+    basic run class to measure execution time. Either call this class calling it by 'with' or delete the class instance
+    after finishing the measurement. The duration result is logged.
+    """
+
+    def __init__(self):
+        self.time = TimeTracking()
+        logging.info(f"{self.__class__.__name__} started")
+
+    def __del__(self):
+        self.time.stop()
+        logging.info(f"{self.__class__.__name__} finished after {self.time}")
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass
+
+    def do_stuff(self):
+        time.sleep(2)
+
+
+class PreProcessing(run):
+
+    def __init__(self, setup):
+        super().__init__()
+        self.setup = setup
+
+
+class Training(run):
+
+    def __init__(self, setup):
+        super().__init__()
+        self.setup = setup
+
+
+class PostProcessing(run):
+
+    def __init__(self, setup):
+        super().__init__()
+        self.setup = setup