diff --git a/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh b/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh index aa84de9de7dce7015b26f040aaec48d0b096a816..16ad4174d7d3dd345aa668b5feab5069ad402f71 100644 --- a/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh +++ b/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh @@ -12,6 +12,10 @@ #SBATCH --mail-type=ALL #SBATCH --mail-user=b.gong@fz-juelich.de +######### Template identifier (don't remove) ######### +echo "Do not run the template scripts" +exit 99 +######### Template identifier (don't remove) ######### # Name of virtual environment VIRT_ENV_NAME="virt_env_hdfml" @@ -29,34 +33,29 @@ fi # Loading mouldes source ../env_setup/modules_preprocess.sh -source_dir=${SAVE_DIR}/extractedData -destination_dir=${SAVE_DIR}/preprocessedData/era5-Y2015to2017M01to12 -script_dir=`pwd` - -declare -a years=("2222" - "2010_1" - "2012" - "2013_complete" - "2015" - "2016" - "2017" - "2019" - ) - +# select years for dataset declare -a years=( "2015" "2016" "2017" ) +max_year=`echo "${years[*]}" | sort -nr | head -n1` +min_year=`echo "${years[*]}" | sort -nr | tail -n1` +# set some paths +# note, that destination_dir is used during runtime to set a proper experiment directory +exp_id=xxx +source_dir=${SAVE_DIR}/extractedData +destination_dir=${SAVE_DIR}/preprocessedData/era5-Y${min_year}to${max_year}M01to12 +script_dir=`pwd` -# ececute Python-scripts +# execute Python-scripts for year in "${years[@]}"; do echo "Year $year" echo "source_dir ${source_dir}/${year}" srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \ - --source_dir ${source_dir} -scr_dir ${script_dir} \ + --source_dir ${source_dir} -scr_dir ${script_dir} --exp_id ${exp_id} \ --destination_dir ${destination_dir} --years ${year} --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710 done