From c2913215ee1df5ec5b575f740c4e616f2f585689 Mon Sep 17 00:00:00 2001
From: lukas leufen <l.leufen@fz-juelich.de>
Date: Mon, 29 Jun 2020 18:05:20 +0200
Subject: [PATCH] updated all run scripts with new Workflow class, HPC_setup
script is calling now the new run_HPC.py script
---
HPC_setup/create_runscripts_HPC.sh | 2 +-
run.py | 27 +++--------------
run_HPC.py | 19 ++++++++++++
run_hourly.py | 21 ++-----------
run_zam347.py | 13 ++------
src/run.py | 16 ++++------
src/workflows/__init__.py | 2 +-
src/workflows/default_workflow.py | 48 ++++++++++++++++++++++++++++--
8 files changed, 83 insertions(+), 65 deletions(-)
create mode 100644 run_HPC.py
diff --git a/HPC_setup/create_runscripts_HPC.sh b/HPC_setup/create_runscripts_HPC.sh
index 1322f47d..bcbb5fb0 100755
--- a/HPC_setup/create_runscripts_HPC.sh
+++ b/HPC_setup/create_runscripts_HPC.sh
@@ -112,7 +112,7 @@ timestamp=\`date +"%Y-%m-%d_%H%M-%S"\`
export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.6/site-packages:\${PYTHONPATH}
-srun python run.py --experiment_date=\$timestamp
+srun python run_HPC.py --experiment_date=\$timestamp
EOT
fi
diff --git a/run.py b/run.py
index 2395064c..a9d81906 100644
--- a/run.py
+++ b/run.py
@@ -1,38 +1,19 @@
__author__ = "Lukas Leufen"
-__date__ = '2019-11-14'
+__date__ = '2020-06-29'
import argparse
-
-from src.run_modules.experiment_setup import ExperimentSetup
-from src.run_modules.partition_check import PartitionCheck
-from src.run_modules.model_setup import ModelSetup
-from src.run_modules.post_processing import PostProcessing
-from src.run_modules.pre_processing import PreProcessing
-from src.run_modules.run_environment import RunEnvironment
-from src.run_modules.training import Training
+from src.workflows import DefaultWorkflow
def main(parser_args):
- experiment_date = parser_args.experiment_date
- with RunEnvironment():
- ExperimentSetup(stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBW001'],
- station_type='background', trainable=False, create_new_model=True, window_history_size=6,
- experiment_date=experiment_date, create_new_bootstraps=False)
- PreProcessing()
-
- PartitionCheck()
-
- ModelSetup()
- Training()
+ workflow = DefaultWorkflow(**parser_args.__dict__)
+ workflow.run()
- PostProcessing()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--experiment_date', metavar='--exp_date', type=str, default="testrun",
help="set experiment date as string")
args = parser.parse_args()
-
main(args)
-
diff --git a/run_HPC.py b/run_HPC.py
new file mode 100644
index 00000000..fc2ead40
--- /dev/null
+++ b/run_HPC.py
@@ -0,0 +1,19 @@
+__author__ = "Lukas Leufen"
+__date__ = '2020-06-29'
+
+import argparse
+from src.workflows import DefaultWorkflowHPC
+
+
+def main(parser_args):
+
+ workflow = DefaultWorkflowHPC(**parser_args.__dict__)
+ workflow.run()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--experiment_date', metavar='--exp_date', type=str, default="testrun",
+ help="set experiment date as string")
+ args = parser.parse_args()
+ main(args)
diff --git a/run_hourly.py b/run_hourly.py
index df326640..682988f6 100644
--- a/run_hourly.py
+++ b/run_hourly.py
@@ -3,27 +3,13 @@ __date__ = '2019-11-14'
import argparse
-from src.run_modules.experiment_setup import ExperimentSetup
-from src.run_modules.model_setup import ModelSetup
-from src.run_modules.post_processing import PostProcessing
-from src.run_modules.pre_processing import PreProcessing
-from src.run_modules.run_environment import RunEnvironment
-from src.run_modules.training import Training
+from src.workflows import DefaultWorkflow
def main(parser_args):
- experiment_date = parser_args.experiment_date
- with RunEnvironment():
- ExperimentSetup(stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBW001'],
- station_type='background', trainable=True, sampling="hourly", window_history_size=48,
- experiment_date=experiment_date)
- PreProcessing()
- ModelSetup()
-
- Training()
-
- PostProcessing()
+ workflow = DefaultWorkflow(sampling="hourly", window_history_size=48, **parser_args.__dict__)
+ workflow.run()
if __name__ == "__main__":
@@ -31,5 +17,4 @@ if __name__ == "__main__":
parser.add_argument('--experiment_date', metavar='--exp_date', type=str, default=None,
help="set experiment date as string")
args = parser.parse_args(["--experiment_date", "testrun"])
-
main(args)
diff --git a/run_zam347.py b/run_zam347.py
index 69b3cd6f..2d351a89 100644
--- a/run_zam347.py
+++ b/run_zam347.py
@@ -11,6 +11,7 @@ from src.run_modules.post_processing import PostProcessing
from src.run_modules.pre_processing import PreProcessing
from src.run_modules.run_environment import RunEnvironment
from src.run_modules.training import Training
+from src.workflows import DefaultWorkflowHPC
def load_stations():
@@ -29,17 +30,9 @@ def load_stations():
def main(parser_args):
- experiment_date = parser_args.experiment_date
- with RunEnvironment():
- ExperimentSetup(stations=load_stations(), station_type='background', trainable=False, create_new_model=True,
- experiment_date=experiment_date)
- PreProcessing()
- ModelSetup()
-
- Training()
-
- PostProcessing()
+ workflow = DefaultWorkflowHPC(stations=load_stations(), **parser_args.__dict__)
+ workflow.run()
if __name__ == "__main__":
diff --git a/src/run.py b/src/run.py
index 900ebb47..1494be0a 100644
--- a/src/run.py
+++ b/src/run.py
@@ -1,6 +1,7 @@
+__author__ = "Lukas Leufen"
+__date__ = '2020-06-29'
-from src.run_modules import *
-import argparse
+from src.workflows import DefaultWorkflow
import inspect
@@ -29,16 +30,11 @@ def run(stations=None,
batch_size=None,
epochs=None):
- params = inspect.getfullargspec(ExperimentSetup).args
+ params = inspect.getfullargspec(DefaultWorkflow).args
kwargs = {k: v for k, v in locals().items() if k in params and v is not None}
- with RunEnvironment():
- ExperimentSetup(**kwargs)
- PreProcessing()
- PartitionCheck()
- ModelSetup()
- Training()
- PostProcessing()
+ workflow = DefaultWorkflow(**kwargs)
+ workflow.run()
if __name__ == "__main__":
diff --git a/src/workflows/__init__.py b/src/workflows/__init__.py
index 184734ae..57e514cd 100644
--- a/src/workflows/__init__.py
+++ b/src/workflows/__init__.py
@@ -1,2 +1,2 @@
from src.workflows.abstract_workflow import Workflow
-from src.workflows.default_workflow import DefaultWorkflow
\ No newline at end of file
+from src.workflows.default_workflow import DefaultWorkflow, DefaultWorkflowHPC
\ No newline at end of file
diff --git a/src/workflows/default_workflow.py b/src/workflows/default_workflow.py
index d20d8a73..6a60c6ae 100644
--- a/src/workflows/default_workflow.py
+++ b/src/workflows/default_workflow.py
@@ -10,8 +10,52 @@ from src.workflows.abstract_workflow import Workflow
class DefaultWorkflow(Workflow):
- """A default workflow executing ExperimentSetup, PreProcessing, PartitionCheck, ModelSetup, Training and
- PostProcessing in exact the mentioned ordering."""
+ """A default workflow executing ExperimentSetup, PreProcessing, ModelSetup, Training and PostProcessing in exact
+ the mentioned ordering."""
+
+ def __init__(self, stations=None,
+ station_type=None,
+ trainable=None, create_new_model=None,
+ window_history_size=None,
+ experiment_date="testrun",
+ network=None,
+ variables=None, statistics_per_var=None,
+ start=None, end=None,
+ target_var=None, target_dim=None,
+ window_lead_time=None,
+ dimensions=None,
+ interpolate_method=None, interpolate_dim=None, limit_nan_fill=None,
+ train_start=None, train_end=None, val_start=None, val_end=None, test_start=None, test_end=None,
+ use_all_stations_on_all_data_sets=None, fraction_of_train=None,
+ experiment_path=None, plot_path=None, forecast_path=None, bootstrap_path=None, overwrite_local_data=None,
+ sampling=None,
+ permute_data_on_training=None, extreme_values=None, extremes_on_right_tail_only=None,
+ transformation=None,
+ train_min_length=None, val_min_length=None, test_min_length=None,
+ evaluate_bootstraps=None, number_of_bootstraps=None, create_new_bootstraps=None,
+ plot_list=None,
+ model=None,
+ batch_size=None,
+ epochs=None):
+ super().__init__()
+
+ # extract all given kwargs arguments
+ params = remove_items(inspect.getfullargspec(self.__init__).args, "self")
+ kwargs = {k: v for k, v in locals().items() if k in params and v is not None}
+ self._setup(**kwargs)
+
+ def _setup(self, **kwargs):
+ """Set up default workflow."""
+ self.add(ExperimentSetup, **kwargs)
+ self.add(PreProcessing)
+ self.add(ModelSetup)
+ self.add(Training)
+ self.add(PostProcessing)
+
+
+class DefaultWorkflowHPC(Workflow):
+ """A default workflow for Jülich HPC systems executing ExperimentSetup, PreProcessing, PartitionCheck, ModelSetup,
+ Training and PostProcessing in exact the mentioned ordering."""
def __init__(self, stations=None,
station_type=None,
--
GitLab