diff --git a/src/__init__.py b/src/__init__.py index 9559822193d070bb886b59e2605bfc7aa73eef5e..c013fb2632cd9201e87603f5c3757b704461eea3 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -5,7 +5,7 @@ __version_info__ = { } from src.run_modules import * -from src.run import run +from src.workflows.default_workflow import DefaultWorkflow def get_version(): diff --git a/src/configuration/defaults.py b/src/configuration/defaults.py index 7ce96cfce515e7f32d98444e6a9542c9fbd7b4f4..0038bb5512d602150905f6504bcd5e135b127382 100644 --- a/src/configuration/defaults.py +++ b/src/configuration/defaults.py @@ -2,12 +2,7 @@ __author__ = "Lukas Leufen" __date__ = '2020-06-25' -DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004', - 'DEBY020', 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073', 'DEBY039', 'DEBW038', 'DEBW081', - 'DEBY075', 'DEBW040', 'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', 'DEBW042', 'DEBW039', 'DEBY001', - 'DEBY113', 'DEBY089', 'DEBW024', 'DEBW004', 'DEBY037', 'DEBW056', 'DEBW029', 'DEBY068', 'DEBW010', - 'DEBW026', 'DEBY002', 'DEBY079', 'DEBW084', 'DEBY049', 'DEBY031', 'DEBW019', 'DEBW001', 'DEBY063', - 'DEBY005', 'DEBW046', 'DEBW103', 'DEBW052', 'DEBW034', 'DEBY088', ] +DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'] DEFAULT_VAR_ALL_DICT = {'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum', 'u': 'average_values', 'v': 'average_values', 'no': 'dma8eu', 'no2': 'dma8eu', 'cloudcover': 'average_values', 'pblheight': 'maximum'} diff --git a/src/workflows/__init__.py b/src/workflows/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/workflows/abstract_workflow.py b/src/workflows/abstract_workflow.py new file mode 100644 index 0000000000000000000000000000000000000000..75b1ea04a945b8652c5ac563e6884c8caa6df97f --- /dev/null +++ b/src/workflows/abstract_workflow.py @@ -0,0 +1,29 @@ +"""Abstract workflow.""" + +__author__ = "Lukas Leufen" +__date__ = '2020-06-26' + +from collections import OrderedDict + +from src import RunEnvironment + + +class AbstractWorkflow: + """Abstract workflow class to handle sequence of stages (run modules). An inheriting class has to first initialise + this mother class and can afterwards add an arbitrary number of stages by using the add method. The execution order + is equal to the ordering of the stages have been added. To run the workflow, finally, a single call of the run + method is sufficient. It must be taken care for inter-stage dependencies, this workflow class only handles the + execution but not the dependencies (workflow would probably fail in this case).""" + + def __init__(self): + self._registry = OrderedDict() + + def add(self, stage, **kwargs): + """Add a new stage with optional kwargs.""" + self._registry[stage] = kwargs + + def run(self): + """Run workflow embedded in a run environment and according to the stage's ordering.""" + with RunEnvironment(): + for stage, kwargs in self._registry.items(): + stage(**kwargs) \ No newline at end of file diff --git a/src/workflows/default_workflow.py b/src/workflows/default_workflow.py new file mode 100644 index 0000000000000000000000000000000000000000..8f87c4daf3005bb37157c02ab09beeb6c3757090 --- /dev/null +++ b/src/workflows/default_workflow.py @@ -0,0 +1,54 @@ +"""Default workflow.""" + +__author__ = "Lukas Leufen" +__date__ = '2020-06-26' + +import inspect +from src.helpers import remove_items +from src.run_modules import ExperimentSetup, PreProcessing, PartitionCheck, ModelSetup, Training, PostProcessing +from src.workflows.abstract_workflow import AbstractWorkflow + + +class DefaultWorkflow(AbstractWorkflow): + """A default workflow executing ExperimentSetup, PreProcessing, PartitionCheck, ModelSetup, Training and + PostProcessing in exact the mentioned ordering.""" + + def __init__(self, stations=None, + station_type=None, + trainable=None, create_new_model=None, + window_history_size=None, + experiment_date="testrun", + network=None, + variables=None, statistics_per_var=None, + start=None, end=None, + target_var=None, target_dim=None, + window_lead_time=None, + dimensions=None, + interpolate_method=None, interpolate_dim=None, limit_nan_fill=None, + train_start=None, train_end=None, val_start=None, val_end=None, test_start=None, test_end=None, + use_all_stations_on_all_data_sets=None, fraction_of_train=None, + experiment_path=None, plot_path=None, forecast_path=None, bootstrap_path=None, overwrite_local_data=None, + sampling=None, + permute_data_on_training=None, extreme_values=None, extremes_on_right_tail_only=None, + transformation=None, + train_min_length=None, val_min_length=None, test_min_length=None, + evaluate_bootstraps=None, number_of_bootstraps=None, create_new_bootstraps=None, + plot_list=None, + model=None, + batch_size=None, + epochs=None): + super().__init__() + + # extract all given kwargs arguments + params = remove_items(inspect.getfullargspec(self.__init__).args, "self") + kwargs = {k: v for k, v in locals().items() if k in params and v is not None} + self._setup(**kwargs) + + def _setup(self, **kwargs): + """Set up default workflow.""" + self.add(ExperimentSetup, **kwargs) + self.add(PreProcessing) + self.add(PartitionCheck) + self.add(ModelSetup) + self.add(Training) + self.add(PostProcessing)