Skip to content
Snippets Groups Projects
Commit 8517f053 authored by lukas leufen's avatar lukas leufen
Browse files

restructured experiment setup, therefore deactivated some functionality in... 

restructured experiment setup, therefore deactivated some functionality in run.py and modules.py (tests fill probably fail now!)
parent f1e40b9b
Branches
Tags
2 merge requests!17update to v0.4.0,!15new feat split subsets
Show whitespace changes
Inline Side-by-side
run.py
+ 3
− 1
View file @ 8517f053
...@@ -30,4 +30,6 @@ if __name__ == "__main__": ...@@ -30,4 +30,6 @@ if __name__ == "__main__":
help="set experiment date as string") help="set experiment date as string")
args = parser.parse_args() args = parser.parse_args()
main() experiment = ExperimentSetup(args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
a = 1
# main()
src/experiment_setup.py
+ 66
− 70
View file @ 8517f053
__author__ = "Lukas Leufen" __author__ = "Lukas Leufen"
__date__ = '2019-11-15' __date__ = '2019-11-15'
from src import helpers, modules
from src import helpers
import logging import logging
import argparse import argparse
DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004',
'DEBY020', 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073', 'DEBY039', 'DEBW038', 'DEBW081',
'DEBY075', 'DEBW040', 'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', 'DEBW042', 'DEBW039', 'DEBY001',
'DEBY113', 'DEBY089', 'DEBW024', 'DEBW004', 'DEBY037', 'DEBW056', 'DEBW029', 'DEBY068', 'DEBW010',
'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084', 'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063',
'DEBY005', 'DEBW046', 'DEBW103', 'DEBW052', 'DEBW034', 'DEBY088', ]
DEFAULT_VAR_ALL_DICT = {'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum', 'u': 'average_values',
'v': 'average_values', 'no': 'dma8eu', 'no2': 'dma8eu', 'cloudcover': 'average_values',
'pblheight': 'maximum'}
class ExperimentSetup(object): class ExperimentSetup(modules.run):
""" """
params: params:
trainable: Train new model if true, otherwise try to load existing model trainable: Train new model if true, otherwise try to load existing model
""" """
def __init__(self, parser_args, **kwargs): def __init__(self, parser_args, var_all_dict=None, stations=None, network=None, variables=None, target_var="o3",
self.args = self._set_parser_args(parser_args) target_dim=None, dimensions=None, interpolate_dim=None, train_start=None, train_end=None,
self.data_path = None val_start=None, val_end=None, test_start=None, test_end=None, use_all_stations_on_all_data_sets=True,
self.experiment_path = None trainable=False, fraction_of_train=None):
self.experiment_name = None
self.trainable = None
self.fraction_of_train = None
self.use_all_stations_on_all_data_sets = None
self.network = None
self.var_all_dict = None
self.stations = None
self.variables = None
self.dimensions = None
self.interpolate_dim = None
self.target_dim = None
self.target_var = None
self.train_kwargs = None
self.val_kwargs = None
self.test_kwargs = None
self.setup_experiment(**kwargs)
def _set_param(self, param, value, default=None):
if default is not None:
value = value.get(param, default)
setattr(self, param, value)
logging.debug(f"set experiment attribute: {param}={value}")
@staticmethod # create run framework
def _set_parser_args(args): super().__init__()
"""
Transform args to dict if given as argparse.Namespace
:param args:
:return:
"""
if isinstance(args, argparse.Namespace):
return args.__dict__
return args
def setup_experiment(self, **kwargs): # experiment setup
# set data path of this experiment
self._set_param("data_path", helpers.prepare_host()) self._set_param("data_path", helpers.prepare_host())
self._set_param("trainable", trainable, default=False)
self._set_param("fraction_of_train", fraction_of_train, default=0.8)
# set experiment name # set experiment name
exp_date = self.args.get("experiment_date") exp_date = self._get_parser_args(parser_args).get("experiment_date")
exp_name, exp_path = helpers.set_experiment_name(experiment_date=exp_date) exp_name, exp_path = helpers.set_experiment_name(experiment_date=exp_date)
self._set_param("experiment_name", exp_name) self._set_param("experiment_name", exp_name)
self._set_param("experiment_path", exp_path) self._set_param("experiment_path", exp_path)
helpers.check_path_and_create(self.experiment_path) helpers.check_path_and_create(self.data_store.get("experiment_path", "general"))
# setup for data
self._set_param("var_all_dict", var_all_dict, default=DEFAULT_VAR_ALL_DICT)
self._set_param("stations", stations, default=DEFAULT_STATIONS)
self._set_param("network", network, default="AIRBASE")
self._set_param("variables", variables, default=list(self.data_store.get("var_all_dict", "general").keys()))
# target
self._set_param("target_var", target_var, default="o3")
self._set_param("target_dim", target_dim, default='variables')
# set if model is trainable # interpolation
self._set_param("trainable", kwargs, default=True) self._set_param("dimensions", dimensions, default={'new_index': ['datetime', 'Stations']})
self._set_param("interpolate_dim", interpolate_dim, default='datetime')
# set fraction of train # train parameters
self._set_param("fraction_of_train", kwargs, default=0.8) self._set_param("start", train_start, default="1997-01-01", scope="general.train")
self._set_param("end", train_end, default="2007-12-31", scope="general.train")
# validation parameters
self._set_param("start", val_start, default="2008-01-01", scope="general.val")
self._set_param("end", val_end, default="2009-12-31", scope="general.val")
# validation parameters
self._set_param("start", test_start, default="2010-01-01", scope="general.test")
self._set_param("end", test_end, default="2017-12-31", scope="general.test")
# use all stations on all data sets (train, val, test) # use all stations on all data sets (train, val, test)
self._set_param("use_all_stations_on_all_data_sets", kwargs, default=True) self._set_param("use_all_stations_on_all_data_sets", use_all_stations_on_all_data_sets, default=True)
self._set_param("network", kwargs, default="AIRBASE")
self._set_param("var_all_dict", kwargs, def _set_param(self, param, value, default=None, scope="general"):
default={'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum', 'u': 'average_values', if value is None and default is not None:
'v': 'average_values', 'no': 'dma8eu', 'no2': 'dma8eu', 'cloudcover': 'average_values', value = default
'pblheight': 'maximum'}) self.data_store.put(param, value, scope)
self._set_param("stations", kwargs, # setattr(self, param, value)
default=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022', logging.debug(f"set experiment attribute: {param}({scope})={value}")
'DEBY004', 'DEBY020', 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073', 'DEBY039',
'DEBW038', 'DEBW081', 'DEBY075', 'DEBW040', 'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', @staticmethod
'DEBW042', 'DEBW039', 'DEBY001', 'DEBY113', 'DEBY089', 'DEBW024', 'DEBW004', 'DEBY037', def _get_parser_args(args):
'DEBW056', 'DEBW029', 'DEBY068', 'DEBW010', 'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084', """
'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063', 'DEBY005', 'DEBW046', 'DEBW103', Transform args to dict if given as argparse.Namespace
'DEBW052', 'DEBW034', 'DEBY088', ]) :param args:
self._set_param("variables", kwargs, default=list(self.var_all_dict.keys())) :return:
self._set_param("dimensions", kwargs, default={'new_index': ['datetime', 'Stations']}) """
self._set_param("interpolate_dim", kwargs, default='datetime') if isinstance(args, argparse.Namespace):
self._set_param("target_dim", kwargs, default='variables') return args.__dict__
self._set_param("target_var", kwargs, default="o3") return args
self._set_param("train_kwargs", kwargs, default={"start": "1997-01-01", "end": "2007-12-31"})
self._set_param("val_kwargs", kwargs, default={"start": "2008-01-01", "end": "2009-12-31"})
self._set_param("test_kwargs", kwargs, default={"start": "2010-01-01", "end": "2017-12-31"})
src/modules.py
+ 7
− 5
View file @ 8517f053
...@@ -2,7 +2,8 @@ from src.helpers import TimeTracking ...@@ -2,7 +2,8 @@ from src.helpers import TimeTracking
import logging import logging
import time import time
from src.data_generator import DataGenerator from src.data_generator import DataGenerator
from src.experiment_setup import ExperimentSetup # from src.experiment_setup import ExperimentSetup
from src.datastore import DataStoreByScope as DataStoreObject
import argparse import argparse
from typing import Dict, List, Any, Tuple from typing import Dict, List, Any, Tuple
...@@ -14,6 +15,7 @@ class run(object): ...@@ -14,6 +15,7 @@ class run(object):
""" """
del_by_exit = False del_by_exit = False
data_store = DataStoreObject()
def __init__(self): def __init__(self):
""" """
...@@ -52,7 +54,7 @@ class PreProcessing(run): ...@@ -52,7 +54,7 @@ class PreProcessing(run):
testing subsets. testing subsets.
""" """
def __init__(self, experiment_setup: ExperimentSetup): def __init__(self, experiment_setup: Any):
super().__init__() super().__init__()
self.setup = experiment_setup self.setup = experiment_setup
self.kwargs = None self.kwargs = None
...@@ -190,6 +192,6 @@ if __name__ == "__main__": ...@@ -190,6 +192,6 @@ if __name__ == "__main__":
parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None, parser.add_argument('--experiment_date', metavar='--exp_date', type=str, nargs=1, default=None,
help="set experiment date as string") help="set experiment date as string")
parser_args = parser.parse_args() parser_args = parser.parse_args()
with run(): # with run():
setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087']) # setup = ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'])
PreProcessing(setup) # PreProcessing(setup)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment