From 3898f55ea882d9cca1b96fa83170f49f5922f8cc Mon Sep 17 00:00:00 2001
From: Michael <m.langguth@fz-juelich.de>
Date: Tue, 8 Sep 2020 22:03:27 +0200
Subject: [PATCH] Rename workflow scripts to template scripts and add first
 adaptions in set-up script of virtual environment.

---
 ...traction.sh => DataExtraction_template.sh} |  4 +++
 ...to_tf.sh => DataPreprocess2tf_template.sh} |  0
 ...eprocess.sh => DataPreprocess_template.sh} |  0
 ...rate_era5.sh => generate_era5_template.sh} |  0
 .../{train_era5.sh => train_era5_template.sh} |  0
 ...traction.sh => DataExtraction_template.sh} |  0
 ...to_tf.sh => DataPreprocess2tf_template.sh} |  0
 ...eprocess.sh => DataPreprocess_template.sh} |  0
 ...rate_era5.sh => generate_era5_template.sh} |  0
 .../{train_era5.sh => train_era5_template.sh} |  0
 video_prediction_savp/env_setup/create_env.sh | 26 +++++++++++++++++++
 11 files changed, 30 insertions(+)
 rename video_prediction_savp/HPC_scripts/{DataExtraction.sh => DataExtraction_template.sh} (90%)
 mode change 100755 => 100644
 rename video_prediction_savp/HPC_scripts/{DataPreprocess_to_tf.sh => DataPreprocess2tf_template.sh} (100%)
 mode change 100755 => 100644
 rename video_prediction_savp/HPC_scripts/{DataPreprocess.sh => DataPreprocess_template.sh} (100%)
 mode change 100755 => 100644
 rename video_prediction_savp/HPC_scripts/{generate_era5.sh => generate_era5_template.sh} (100%)
 mode change 100755 => 100644
 rename video_prediction_savp/HPC_scripts/{train_era5.sh => train_era5_template.sh} (100%)
 mode change 100755 => 100644
 rename video_prediction_savp/Zam347_scripts/{DataExtraction.sh => DataExtraction_template.sh} (100%)
 mode change 100755 => 100644
 rename video_prediction_savp/Zam347_scripts/{DataPreprocess_to_tf.sh => DataPreprocess2tf_template.sh} (100%)
 mode change 100755 => 100644
 rename video_prediction_savp/Zam347_scripts/{DataPreprocess.sh => DataPreprocess_template.sh} (100%)
 mode change 100755 => 100644
 rename video_prediction_savp/Zam347_scripts/{generate_era5.sh => generate_era5_template.sh} (100%)
 mode change 100755 => 100644
 rename video_prediction_savp/Zam347_scripts/{train_era5.sh => train_era5_template.sh} (100%)
 mode change 100755 => 100644

diff --git a/video_prediction_savp/HPC_scripts/DataExtraction.sh b/video_prediction_savp/HPC_scripts/DataExtraction_template.sh
old mode 100755
new mode 100644
similarity index 90%
rename from video_prediction_savp/HPC_scripts/DataExtraction.sh
rename to video_prediction_savp/HPC_scripts/DataExtraction_template.sh
index b44065e7..a80a3b87
--- a/video_prediction_savp/HPC_scripts/DataExtraction.sh
+++ b/video_prediction_savp/HPC_scripts/DataExtraction_template.sh
@@ -12,6 +12,10 @@
 #SBATCH --mail-type=ALL
 #SBATCH --mail-user=b.gong@fz-juelich.de
 
+######### Template identifier (don't remove) #########
+echo "Do not run the template scripts"
+exit 99
+######### Template identifier (don't remove) #########
 
 jutil env activate -p deepacf
 
diff --git a/video_prediction_savp/HPC_scripts/DataPreprocess_to_tf.sh b/video_prediction_savp/HPC_scripts/DataPreprocess2tf_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/HPC_scripts/DataPreprocess_to_tf.sh
rename to video_prediction_savp/HPC_scripts/DataPreprocess2tf_template.sh
diff --git a/video_prediction_savp/HPC_scripts/DataPreprocess.sh b/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/HPC_scripts/DataPreprocess.sh
rename to video_prediction_savp/HPC_scripts/DataPreprocess_template.sh
diff --git a/video_prediction_savp/HPC_scripts/generate_era5.sh b/video_prediction_savp/HPC_scripts/generate_era5_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/HPC_scripts/generate_era5.sh
rename to video_prediction_savp/HPC_scripts/generate_era5_template.sh
diff --git a/video_prediction_savp/HPC_scripts/train_era5.sh b/video_prediction_savp/HPC_scripts/train_era5_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/HPC_scripts/train_era5.sh
rename to video_prediction_savp/HPC_scripts/train_era5_template.sh
diff --git a/video_prediction_savp/Zam347_scripts/DataExtraction.sh b/video_prediction_savp/Zam347_scripts/DataExtraction_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/Zam347_scripts/DataExtraction.sh
rename to video_prediction_savp/Zam347_scripts/DataExtraction_template.sh
diff --git a/video_prediction_savp/Zam347_scripts/DataPreprocess_to_tf.sh b/video_prediction_savp/Zam347_scripts/DataPreprocess2tf_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/Zam347_scripts/DataPreprocess_to_tf.sh
rename to video_prediction_savp/Zam347_scripts/DataPreprocess2tf_template.sh
diff --git a/video_prediction_savp/Zam347_scripts/DataPreprocess.sh b/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/Zam347_scripts/DataPreprocess.sh
rename to video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh
diff --git a/video_prediction_savp/Zam347_scripts/generate_era5.sh b/video_prediction_savp/Zam347_scripts/generate_era5_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/Zam347_scripts/generate_era5.sh
rename to video_prediction_savp/Zam347_scripts/generate_era5_template.sh
diff --git a/video_prediction_savp/Zam347_scripts/train_era5.sh b/video_prediction_savp/Zam347_scripts/train_era5_template.sh
old mode 100755
new mode 100644
similarity index 100%
rename from video_prediction_savp/Zam347_scripts/train_era5.sh
rename to video_prediction_savp/Zam347_scripts/train_era5_template.sh
diff --git a/video_prediction_savp/env_setup/create_env.sh b/video_prediction_savp/env_setup/create_env.sh
index ad388826..33919c87 100644
--- a/video_prediction_savp/env_setup/create_env.sh
+++ b/video_prediction_savp/env_setup/create_env.sh
@@ -19,6 +19,10 @@ if [[ ! -n "$1" ]]; then
   return
 fi
 
+# list of (Batch) scripts used for the steps in the workflow
+# !!! Expects that a template named [script_name]_template.sh exists!!!
+workflow_scripts=(DataExtraction DataPreprocess DataPreprocess2tf train_era5 generate_era5)
+
 HOST_NAME=`hostname`
 ENV_NAME=$1
 ENV_SETUP_DIR=`pwd`
@@ -86,6 +90,28 @@ if [[ "$ENV_EXIST" == 0 ]]; then
     pip3 install h5py
     pip3 install tensorflow-gpu==1.13.1
   fi
+
+  # After checking and setting up the system, create user-specific runscripts for all steps of the workflow
+  if [[ "${HOST_NAME}" == hdfml* || "${HOST_NAME}" == juwels* ]]; then
+    echo "***** Creating Batch-scripts for running workflow... *****"
+    script_dir=../HPC_scripts
+    for wf_script in "${workflow_scripts[@]}"; do
+      curr_script=${script_dir}/${wf_script}
+      if ! [[ -f ${curr_script}_template.sh ]]; then
+        echo "WARNING: Could not find expected Batch script '${curr_script}_template.sh'."
+        echo "Thus, no corresponding executable script is created!"
+      else
+        cp ${curr_script}_template.sh ${curr_script}_op.sh
+        # remove template identifiers
+        num_lines=`awk '/Template identifiers/{ print NR }'
+        line_s=`echo ${num_lines} | cut -d' ' -f 1`
+        line_e=`echo ${num_lines} | cut -d' ' -f 2`
+        sed -e '${line_s},${line_e}d' ${curr_script}_op.sh
+        # set correct e-mail address
+        sed -i "s/--mail-user=.*/--mail-user=$USER_EMAIL/g" ${curr_script}_op.sh
+      fi
+    done
+
   # expand PYTHONPATH...
   export PYTHONPATH=${WORKING_DIR}:$PYTHONPATH >> ${activate_virt_env}
   #export PYTHONPATH=/p/home/jusers/${USER}/juwels/.local/bin:$PYTHONPATH
-- 
GitLab