diff --git a/video_prediction_savp/HPC_scripts/DataExtraction.sh b/video_prediction_savp/HPC_scripts/DataExtraction_template.sh old mode 100755 new mode 100644 similarity index 90% rename from video_prediction_savp/HPC_scripts/DataExtraction.sh rename to video_prediction_savp/HPC_scripts/DataExtraction_template.sh index b44065e7babb0411cda6d2849ec429f3672c60d5..a80a3b8779908fc51121c6682817f20ec197a327 --- a/video_prediction_savp/HPC_scripts/DataExtraction.sh +++ b/video_prediction_savp/HPC_scripts/DataExtraction_template.sh @@ -12,6 +12,10 @@ #SBATCH --mail-type=ALL #SBATCH --mail-user=b.gong@fz-juelich.de +######### Template identifier (don't remove) ######### +echo "Do not run the template scripts" +exit 99 +######### Template identifier (don't remove) ######### jutil env activate -p deepacf diff --git a/video_prediction_savp/HPC_scripts/DataPreprocess_to_tf.sh b/video_prediction_savp/HPC_scripts/DataPreprocess2tf_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/HPC_scripts/DataPreprocess_to_tf.sh rename to video_prediction_savp/HPC_scripts/DataPreprocess2tf_template.sh diff --git a/video_prediction_savp/HPC_scripts/DataPreprocess.sh b/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/HPC_scripts/DataPreprocess.sh rename to video_prediction_savp/HPC_scripts/DataPreprocess_template.sh diff --git a/video_prediction_savp/HPC_scripts/generate_era5.sh b/video_prediction_savp/HPC_scripts/generate_era5_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/HPC_scripts/generate_era5.sh rename to video_prediction_savp/HPC_scripts/generate_era5_template.sh diff --git a/video_prediction_savp/HPC_scripts/train_era5.sh b/video_prediction_savp/HPC_scripts/train_era5_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/HPC_scripts/train_era5.sh rename to video_prediction_savp/HPC_scripts/train_era5_template.sh diff --git a/video_prediction_savp/Zam347_scripts/DataExtraction.sh b/video_prediction_savp/Zam347_scripts/DataExtraction_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/Zam347_scripts/DataExtraction.sh rename to video_prediction_savp/Zam347_scripts/DataExtraction_template.sh diff --git a/video_prediction_savp/Zam347_scripts/DataPreprocess_to_tf.sh b/video_prediction_savp/Zam347_scripts/DataPreprocess2tf_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/Zam347_scripts/DataPreprocess_to_tf.sh rename to video_prediction_savp/Zam347_scripts/DataPreprocess2tf_template.sh diff --git a/video_prediction_savp/Zam347_scripts/DataPreprocess.sh b/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/Zam347_scripts/DataPreprocess.sh rename to video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh diff --git a/video_prediction_savp/Zam347_scripts/generate_era5.sh b/video_prediction_savp/Zam347_scripts/generate_era5_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/Zam347_scripts/generate_era5.sh rename to video_prediction_savp/Zam347_scripts/generate_era5_template.sh diff --git a/video_prediction_savp/Zam347_scripts/train_era5.sh b/video_prediction_savp/Zam347_scripts/train_era5_template.sh old mode 100755 new mode 100644 similarity index 100% rename from video_prediction_savp/Zam347_scripts/train_era5.sh rename to video_prediction_savp/Zam347_scripts/train_era5_template.sh diff --git a/video_prediction_savp/env_setup/create_env.sh b/video_prediction_savp/env_setup/create_env.sh index ad388826caf1d077c1c6434acae29d6cbaa9c6fc..33919c8752b9b66ed7024ecdb26b50bdd47a24e7 100644 --- a/video_prediction_savp/env_setup/create_env.sh +++ b/video_prediction_savp/env_setup/create_env.sh @@ -19,6 +19,10 @@ if [[ ! -n "$1" ]]; then return fi +# list of (Batch) scripts used for the steps in the workflow +# !!! Expects that a template named [script_name]_template.sh exists!!! +workflow_scripts=(DataExtraction DataPreprocess DataPreprocess2tf train_era5 generate_era5) + HOST_NAME=`hostname` ENV_NAME=$1 ENV_SETUP_DIR=`pwd` @@ -86,6 +90,28 @@ if [[ "$ENV_EXIST" == 0 ]]; then pip3 install h5py pip3 install tensorflow-gpu==1.13.1 fi + + # After checking and setting up the system, create user-specific runscripts for all steps of the workflow + if [[ "${HOST_NAME}" == hdfml* || "${HOST_NAME}" == juwels* ]]; then + echo "***** Creating Batch-scripts for running workflow... *****" + script_dir=../HPC_scripts + for wf_script in "${workflow_scripts[@]}"; do + curr_script=${script_dir}/${wf_script} + if ! [[ -f ${curr_script}_template.sh ]]; then + echo "WARNING: Could not find expected Batch script '${curr_script}_template.sh'." + echo "Thus, no corresponding executable script is created!" + else + cp ${curr_script}_template.sh ${curr_script}_op.sh + # remove template identifiers + num_lines=`awk '/Template identifiers/{ print NR }' + line_s=`echo ${num_lines} | cut -d' ' -f 1` + line_e=`echo ${num_lines} | cut -d' ' -f 2` + sed -e '${line_s},${line_e}d' ${curr_script}_op.sh + # set correct e-mail address + sed -i "s/--mail-user=.*/--mail-user=$USER_EMAIL/g" ${curr_script}_op.sh + fi + done + # expand PYTHONPATH... export PYTHONPATH=${WORKING_DIR}:$PYTHONPATH >> ${activate_virt_env} #export PYTHONPATH=/p/home/jusers/${USER}/juwels/.local/bin:$PYTHONPATH