restructured experiment setup, therefore deactivated some functionality in...

restructured experiment setup, therefore deactivated some functionality in run.py and modules.py (tests fill probably fail now!)

run.py

@@ -30,4 +30,6 @@ if __name__ == "__main__":
                         help="set experiment date as string")
     args = parser.parse_args()
 
-    main()
+    experiment = ExperimentSetup(args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
+    a = 1
+    # main()

src/experiment_setup.py

 __author__ = "Lukas Leufen"
 __date__ = '2019-11-15'
 
-
-from src import helpers
+from src import helpers, modules
 import logging
 import argparse
 
+DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004',
+                    'DEBY020', 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073', 'DEBY039', 'DEBW038', 'DEBW081',
+                    'DEBY075', 'DEBW040', 'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', 'DEBW042', 'DEBW039', 'DEBY001',
+                    'DEBY113', 'DEBY089', 'DEBW024', 'DEBW004', 'DEBY037', 'DEBW056', 'DEBW029', 'DEBY068', 'DEBW010',
+                    'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084', 'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063',
+                    'DEBY005', 'DEBW046', 'DEBW103', 'DEBW052', 'DEBW034', 'DEBY088', ]
+DEFAULT_VAR_ALL_DICT = {'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum', 'u': 'average_values',
+                        'v': 'average_values', 'no': 'dma8eu', 'no2': 'dma8eu', 'cloudcover': 'average_values',
+                        'pblheight': 'maximum'}
+
 
-class ExperimentSetup(object):
+class ExperimentSetup(modules.run):
     """
     params:
     trainable: Train new model if true, otherwise try to load existing model
     """
 
-    def __init__(self, parser_args, **kwargs):
-        self.args = self._set_parser_args(parser_args)
-        self.data_path = None
-        self.experiment_path = None
-        self.experiment_name = None
-        self.trainable = None
-        self.fraction_of_train = None
-        self.use_all_stations_on_all_data_sets = None
-        self.network = None
-        self.var_all_dict = None
-        self.stations = None
-        self.variables = None
-        self.dimensions = None
-        self.interpolate_dim = None
-        self.target_dim = None
-        self.target_var = None
-        self.train_kwargs = None
-        self.val_kwargs = None
-        self.test_kwargs = None
-        self.setup_experiment(**kwargs)
-
-    def _set_param(self, param, value, default=None):
-        if default is not None:
-            value = value.get(param, default)
-        setattr(self, param, value)
-        logging.debug(f"set experiment attribute: {param}={value}")
-
-    @staticmethod
-    def _set_parser_args(args):
-        """
-        Transform args to dict if given as argparse.Namespace
-        :param args:
-        :return:
-        """
-        if isinstance(args, argparse.Namespace):
-            return args.__dict__
-        return args
+    def __init__(self, parser_args, var_all_dict=None, stations=None, network=None, variables=None, target_var="o3",
+                 target_dim=None, dimensions=None, interpolate_dim=None, train_start=None, train_end=None,
+                 val_start=None, val_end=None, test_start=None, test_end=None, use_all_stations_on_all_data_sets=True,
+                 trainable=False, fraction_of_train=None):
 
-    def setup_experiment(self, **kwargs):
+        # create run framework
+        super().__init__()
 
-        # set data path of this experiment
+        # experiment setup
         self._set_param("data_path", helpers.prepare_host())
+        self._set_param("trainable", trainable, default=False)
+        self._set_param("fraction_of_train", fraction_of_train, default=0.8)
 
         # set experiment name
-        exp_date = self.args.get("experiment_date")
+        exp_date = self._get_parser_args(parser_args).get("experiment_date")
         exp_name, exp_path = helpers.set_experiment_name(experiment_date=exp_date)
         self._set_param("experiment_name", exp_name)
         self._set_param("experiment_path", exp_path)
-        helpers.check_path_and_create(self.experiment_path)
+        helpers.check_path_and_create(self.data_store.get("experiment_path", "general"))
+
+        # setup for data
+        self._set_param("var_all_dict", var_all_dict, default=DEFAULT_VAR_ALL_DICT)
+        self._set_param("stations", stations, default=DEFAULT_STATIONS)
+        self._set_param("network", network, default="AIRBASE")
+        self._set_param("variables", variables, default=list(self.data_store.get("var_all_dict", "general").keys()))
+
+        # target
+        self._set_param("target_var", target_var, default="o3")
+        self._set_param("target_dim", target_dim, default='variables')
 
-        # set if model is trainable
-        self._set_param("trainable", kwargs, default=True)
+        # interpolation
+        self._set_param("dimensions", dimensions, default={'new_index': ['datetime', 'Stations']})
+        self._set_param("interpolate_dim", interpolate_dim, default='datetime')
 
-        # set fraction of train
-        self._set_param("fraction_of_train", kwargs, default=0.8)
+        # train parameters
+        self._set_param("start", train_start, default="1997-01-01", scope="general.train")
+        self._set_param("end", train_end, default="2007-12-31", scope="general.train")
+
+        # validation parameters
+        self._set_param("start", val_start, default="2008-01-01", scope="general.val")
+        self._set_param("end", val_end, default="2009-12-31", scope="general.val")
+
+        # validation parameters
+        self._set_param("start", test_start, default="2010-01-01", scope="general.test")
+        self._set_param("end", test_end, default="2017-12-31", scope="general.test")
 
         # use all stations on all data sets (train, val, test)
-        self._set_param("use_all_stations_on_all_data_sets", kwargs, default=True)
-        self._set_param("network", kwargs, default="AIRBASE")
-        self._set_param("var_all_dict", kwargs,
-                        default={'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum', 'u': 'average_values',
-                                 'v': 'average_values', 'no': 'dma8eu', 'no2': 'dma8eu', 'cloudcover': 'average_values',
-                                 'pblheight': 'maximum'})
-        self._set_param("stations", kwargs,
-                        default=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022',
-                                 'DEBY004', 'DEBY020', 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073', 'DEBY039',
-                                 'DEBW038', 'DEBW081', 'DEBY075', 'DEBW040', 'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072',
-                                 'DEBW042', 'DEBW039', 'DEBY001', 'DEBY113', 'DEBY089', 'DEBW024', 'DEBW004', 'DEBY037',
-                                 'DEBW056', 'DEBW029', 'DEBY068', 'DEBW010', 'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084',
-                                 'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063', 'DEBY005', 'DEBW046', 'DEBW103',
-                                 'DEBW052', 'DEBW034', 'DEBY088', ])
-        self._set_param("variables", kwargs, default=list(self.var_all_dict.keys()))
-        self._set_param("dimensions", kwargs, default={'new_index': ['datetime', 'Stations']})
-        self._set_param("interpolate_dim", kwargs, default='datetime')
-        self._set_param("target_dim", kwargs, default='variables')
-        self._set_param("target_var", kwargs, default="o3")
-        self._set_param("train_kwargs", kwargs, default={"start": "1997-01-01", "end": "2007-12-31"})
-        self._set_param("val_kwargs", kwargs, default={"start": "2008-01-01", "end": "2009-12-31"})
-        self._set_param("test_kwargs", kwargs, default={"start": "2010-01-01", "end": "2017-12-31"})
+        self._set_param("use_all_stations_on_all_data_sets", use_all_stations_on_all_data_sets, default=True)
+
+    def _set_param(self, param, value, default=None, scope="general"):
+        if value is None and default is not None:
+            value = default
+        self.data_store.put(param, value, scope)
+        # setattr(self, param, value)
+        logging.debug(f"set experiment attribute: {param}({scope})={value}")
+
+    @staticmethod
+    def _get_parser_args(args):
+        """
+        Transform args to dict if given as argparse.Namespace
+        :param args:
+        :return:
+        """
+        if isinstance(args, argparse.Namespace):
+            return args.__dict__
+        return args

src/modules.py

@@ -2,7 +2,8 @@ from src.helpers import TimeTracking
 import logging
 import time
 from src.data_generator import DataGenerator
-from src.experiment_setup import ExperimentSetup
+# from src.experiment_setup import ExperimentSetup
+from src.datastore import DataStoreByScope as DataStoreObject
 import argparse
 from typing import Dict, List, Any, Tuple
 
@@ -14,6 +15,7 @@ class run(object):
     """
 
     del_by_exit = False
+    data_store = DataStoreObject()
 
     def __init__(self):
         """
@@ -52,7 +54,7 @@ class PreProcessing(run):
     testing subsets.
     """
 
-    def __init__(self, experiment_setup: ExperimentSetup):
+    def __init__(self, experiment_setup: Any):
         super().__init__()
         self.setup = experiment_setup
         self.kwargs = None
@@ -190,6 +192,6 @@ if __name__ == "__main__":
     parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None,
                         help="set experiment date as string")
     parser_args = parser.parse_args()
-    with run():
-        setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
-        PreProcessing(setup)
+    # with run():
+    #     setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
+    #     PreProcessing(setup)