diff --git a/run.py b/run.py
index 6b5c367dae776d7aa1c5f26d70b6b952b64e064d..fb1ef8d9b33fd6f7910afe6c727364e6557d6911 100644
--- a/run.py
+++ b/run.py
@@ -3,133 +3,15 @@ __date__ = '2019-11-14'
import logging
-from src.helpers import TimeTracking
-from src import helpers
import argparse
-import time
+from src.experiment_setup import ExperimentSetup
+from src.modules import run, PreProcessing, Training, PostProcessing
formatter = "%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]"
logging.basicConfig(level=logging.INFO, format=formatter)
-class run(object):
- """
- basic run class to measure execution time. Either call this class calling it by 'with' or delete the class instance
- after finishing the measurement. The duration result is logged.
- """
-
- def __init__(self):
- self.time = TimeTracking()
- logging.info(f"{self.__class__.__name__} started")
-
- def __del__(self):
- self.time.stop()
- logging.info(f"{self.__class__.__name__} finished after {self.time}")
-
- def __enter__(self):
- pass
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- pass
-
- def do_stuff(self):
- time.sleep(2)
-
-
-class ExperimentSetup:
- """
- params:
- trainable: Train new model if true, otherwise try to load existing model
- """
-
- def __init__(self, **kwargs):
- self.data_path = None
- self.experiment_path = None
- self.experiment_name = None
- self.trainable = None
- self.fraction_of_train = None
- self.use_all_stations_on_all_data_sets = None
- self.network = None
- self.var_all_dict = None
- self.all_stations = None
- self.variables = None
- self.dimensions = None
- self.dim = None
- self.target_dim = None
- self.target_var = None
- self.setup_experiment(**kwargs)
-
- def _set_param(self, param, value, default=None):
- if default is not None:
- value = value.get(param, default)
- setattr(self, param, value)
- logging.info(f"set experiment attribute: {param}={value}")
-
- def setup_experiment(self, **kwargs):
-
- # set data path of this experiment
- self._set_param("data_path", helpers.prepare_host())
-
- # set experiment name
- exp_date = args.experiment_date
- exp_name, exp_path = helpers.set_experiment_name(experiment_date=exp_date)
- self._set_param("experiment_name", exp_name)
- self._set_param("experiment_path", exp_path)
- helpers.check_path_and_create(self.experiment_path)
-
- # set if model is trainable
- self._set_param("trainable", kwargs, default=True)
-
- # set fraction of train
- self._set_param("fraction_of_train", kwargs, default=0.8)
-
- # use all stations on all data sets (train, val, test)
- self._set_param("use_all_stations_on_all_data_sets", kwargs, default=True)
- self._set_param("network", kwargs, default="AIRBASE")
- self._set_param("var_all_dict", kwargs, default={'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum',
- 'u': 'average_values', 'v': 'average_values', 'no': 'dma8eu',
- 'no2': 'dma8eu', 'cloudcover': 'average_values',
- 'pblheight': 'maximum'})
- self._set_param("all_stations", kwargs, default=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087',
- 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004', 'DEBY020',
- 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073',
- 'DEBY039', 'DEBW038', 'DEBW081', 'DEBY075', 'DEBW040',
- 'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', 'DEBW042',
- 'DEBW039', 'DEBY001', 'DEBY113', 'DEBY089', 'DEBW024',
- 'DEBW004', 'DEBY037', 'DEBW056', 'DEBW029', 'DEBY068',
- 'DEBW010', 'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084',
- 'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063',
- 'DEBY005', 'DEBW046', 'DEBW103', 'DEBW052', 'DEBW034',
- 'DEBY088', ])
- self._set_param("variables", kwargs, default=list(self.var_all_dict.keys()))
- self._set_param("dimensions", kwargs, default={'new_index': ['datetime', 'Stations']})
- self._set_param("dim", kwargs, default='datetime')
- self._set_param("target_dim", kwargs, default='variables')
- self._set_param("target_var", kwargs, default="o3")
-
-
-class PreProcessing(run):
-
- def __init__(self, setup):
- super().__init__()
- self.setup = setup
-
-
-class Training(run):
-
- def __init__(self, setup):
- super().__init__()
- self.setup = setup
-
-
-class PostProcessing(run):
-
- def __init__(self, setup):
- super().__init__()
- self.setup = setup
-
-
if __name__ == "__main__":
parser = argparse.ArgumentParser()
@@ -138,7 +20,7 @@ if __name__ == "__main__":
args = parser.parse_args()
with run():
- exp_setup = ExperimentSetup(trainable=True)
+ exp_setup = ExperimentSetup(args, trainable=True)
PreProcessing(exp_setup)
diff --git a/src/experiment_setup.py b/src/experiment_setup.py
new file mode 100644
index 0000000000000000000000000000000000000000..18b5f7149dee1d15da9d7f6bfa26ccd938c405b3
--- /dev/null
+++ b/src/experiment_setup.py
@@ -0,0 +1,79 @@
+__author__ = "Lukas Leufen"
+__date__ = '2019-11-15'
+
+
+from src import helpers
+import logging
+
+
+class ExperimentSetup:
+ """
+ params:
+ trainable: Train new model if true, otherwise try to load existing model
+ """
+
+ def __init__(self, parser_args, **kwargs):
+ self.args = parser_args
+ self.data_path = None
+ self.experiment_path = None
+ self.experiment_name = None
+ self.trainable = None
+ self.fraction_of_train = None
+ self.use_all_stations_on_all_data_sets = None
+ self.network = None
+ self.var_all_dict = None
+ self.all_stations = None
+ self.variables = None
+ self.dimensions = None
+ self.dim = None
+ self.target_dim = None
+ self.target_var = None
+ self.setup_experiment(**kwargs)
+
+ def _set_param(self, param, value, default=None):
+ if default is not None:
+ value = value.get(param, default)
+ setattr(self, param, value)
+ logging.debug(f"set experiment attribute: {param}={value}")
+
+ def setup_experiment(self, **kwargs):
+
+ # set data path of this experiment
+ self._set_param("data_path", helpers.prepare_host())
+
+ # set experiment name
+ exp_date = self.args.experiment_date
+ exp_name, exp_path = helpers.set_experiment_name(experiment_date=exp_date)
+ self._set_param("experiment_name", exp_name)
+ self._set_param("experiment_path", exp_path)
+ helpers.check_path_and_create(self.experiment_path)
+
+ # set if model is trainable
+ self._set_param("trainable", kwargs, default=True)
+
+ # set fraction of train
+ self._set_param("fraction_of_train", kwargs, default=0.8)
+
+ # use all stations on all data sets (train, val, test)
+ self._set_param("use_all_stations_on_all_data_sets", kwargs, default=True)
+ self._set_param("network", kwargs, default="AIRBASE")
+ self._set_param("var_all_dict", kwargs, default={'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum',
+ 'u': 'average_values', 'v': 'average_values', 'no': 'dma8eu',
+ 'no2': 'dma8eu', 'cloudcover': 'average_values',
+ 'pblheight': 'maximum'})
+ self._set_param("all_stations", kwargs, default=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087',
+ 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004', 'DEBY020',
+ 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073',
+ 'DEBY039', 'DEBW038', 'DEBW081', 'DEBY075', 'DEBW040',
+ 'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', 'DEBW042',
+ 'DEBW039', 'DEBY001', 'DEBY113', 'DEBY089', 'DEBW024',
+ 'DEBW004', 'DEBY037', 'DEBW056', 'DEBW029', 'DEBY068',
+ 'DEBW010', 'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084',
+ 'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063',
+ 'DEBY005', 'DEBW046', 'DEBW103', 'DEBW052', 'DEBW034',
+ 'DEBY088', ])
+ self._set_param("variables", kwargs, default=list(self.var_all_dict.keys()))
+ self._set_param("dimensions", kwargs, default={'new_index': ['datetime', 'Stations']})
+ self._set_param("dim", kwargs, default='datetime')
+ self._set_param("target_dim", kwargs, default='variables')
+ self._set_param("target_var", kwargs, default="o3")
diff --git a/src/modules.py b/src/modules.py
new file mode 100644
index 0000000000000000000000000000000000000000..04b7f84994772f6ed236afae14b6c7c913867c82
--- /dev/null
+++ b/src/modules.py
@@ -0,0 +1,48 @@
+from src.helpers import TimeTracking
+import logging
+import time
+
+
+class run(object):
+ """
+ basic run class to measure execution time. Either call this class calling it by 'with' or delete the class instance
+ after finishing the measurement. The duration result is logged.
+ """
+
+ def __init__(self):
+ self.time = TimeTracking()
+ logging.info(f"{self.__class__.__name__} started")
+
+ def __del__(self):
+ self.time.stop()
+ logging.info(f"{self.__class__.__name__} finished after {self.time}")
+
+ def __enter__(self):
+ pass
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ pass
+
+ def do_stuff(self):
+ time.sleep(2)
+
+
+class PreProcessing(run):
+
+ def __init__(self, setup):
+ super().__init__()
+ self.setup = setup
+
+
+class Training(run):
+
+ def __init__(self, setup):
+ super().__init__()
+ self.setup = setup
+
+
+class PostProcessing(run):
+
+ def __init__(self, setup):
+ super().__init__()
+ self.setup = setup