diff --git a/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh b/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh index 16ad4174d7d3dd345aa668b5feab5069ad402f71..c19d2ebd5487a689703d12e554c9889d952877d1 100644 --- a/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh +++ b/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh @@ -45,7 +45,7 @@ max_year=`echo "${years[*]}" | sort -nr | head -n1` min_year=`echo "${years[*]}" | sort -nr | tail -n1` # set some paths # note, that destination_dir is used during runtime to set a proper experiment directory -exp_id=xxx +exp_id=xxx # experiment identifier is set by 'generate_workflow_runscripts.sh' source_dir=${SAVE_DIR}/extractedData destination_dir=${SAVE_DIR}/preprocessedData/era5-Y${min_year}to${max_year}M01to12 script_dir=`pwd` diff --git a/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh b/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh index 8f25c2b3dec3140f4a8a77d1a992e2a1682c69ac..172763151557108664c8ba8c9b9028ea2aa5f4f0 100644 --- a/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh +++ b/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh @@ -5,20 +5,26 @@ echo "Do not run the template scripts" exit 99 ######### Template identifier (don't remove) ######### -# some path variables -source_dir=/home/$USER/extractedData -destination_dir=/home/$USER/preprocessedData/era5-Y2017M01to02 # adapt suffix manually! +# select years for dataset +declare -a years=( + "2017" + ) + +max_year=`echo "${years[*]}" | sort -nr | head -n1` +min_year=`echo "${years[*]}" | sort -nr | tail -n1` +# set some paths +# note, that destination_dir is used during runtime to set a proper experiment directory +exp_id=xxx # experiment identifier is set by 'generate_workflow_runscripts.sh' +source_dir=${SAVE_DIR}/extractedData +destination_dir=${SAVE_DIR}/preprocessedData/era5-Y${min_year}to${max_year}M01to12 script_dir=`pwd` -# select years to process -declare -a years=("2017") - for year in "${years[@]}"; do echo "Year $year" echo "source_dir ${source_dir}/${year}" mpirun -np 2 python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \ - --source_dir ${source_dir} -scr_dir ${script_dir} \ + --source_dir ${source_dir} -scr_dir ${script_dir} -exp_dir ${exp_id} \ --destination_dir ${destination_dir} --years ${years} --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710 done python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py --destination_dir ${destination_dir} --varnames T2 MSL gph500