From 53615ed4a4faafd6f8dc7c5a0921afc67e81cfe9 Mon Sep 17 00:00:00 2001
From: Michael <m.langguth@fz-juelich.de>
Date: Thu, 10 Sep 2020 19:00:52 +0200
Subject: [PATCH] Further adaption of runscript 'DataPreprocess_template' for
zam347 although it's expected that the workflow does not run on zam347 right
now.
---
.../HPC_scripts/DataPreprocess_template.sh | 2 +-
.../Zam347_scripts/DataPreprocess_template.sh | 20 ++++++++++++-------
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh b/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh
index 16ad4174..c19d2ebd 100644
--- a/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh
+++ b/video_prediction_savp/HPC_scripts/DataPreprocess_template.sh
@@ -45,7 +45,7 @@ max_year=`echo "${years[*]}" | sort -nr | head -n1`
min_year=`echo "${years[*]}" | sort -nr | tail -n1`
# set some paths
# note, that destination_dir is used during runtime to set a proper experiment directory
-exp_id=xxx
+exp_id=xxx # experiment identifier is set by 'generate_workflow_runscripts.sh'
source_dir=${SAVE_DIR}/extractedData
destination_dir=${SAVE_DIR}/preprocessedData/era5-Y${min_year}to${max_year}M01to12
script_dir=`pwd`
diff --git a/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh b/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh
index 8f25c2b3..17276315 100644
--- a/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh
+++ b/video_prediction_savp/Zam347_scripts/DataPreprocess_template.sh
@@ -5,20 +5,26 @@ echo "Do not run the template scripts"
exit 99
######### Template identifier (don't remove) #########
-# some path variables
-source_dir=/home/$USER/extractedData
-destination_dir=/home/$USER/preprocessedData/era5-Y2017M01to02 # adapt suffix manually!
+# select years for dataset
+declare -a years=(
+ "2017"
+ )
+
+max_year=`echo "${years[*]}" | sort -nr | head -n1`
+min_year=`echo "${years[*]}" | sort -nr | tail -n1`
+# set some paths
+# note, that destination_dir is used during runtime to set a proper experiment directory
+exp_id=xxx # experiment identifier is set by 'generate_workflow_runscripts.sh'
+source_dir=${SAVE_DIR}/extractedData
+destination_dir=${SAVE_DIR}/preprocessedData/era5-Y${min_year}to${max_year}M01to12
script_dir=`pwd`
-# select years to process
-declare -a years=("2017")
-
for year in "${years[@]}";
do
echo "Year $year"
echo "source_dir ${source_dir}/${year}"
mpirun -np 2 python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
- --source_dir ${source_dir} -scr_dir ${script_dir} \
+ --source_dir ${source_dir} -scr_dir ${script_dir} -exp_dir ${exp_id} \
--destination_dir ${destination_dir} --years ${years} --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
done
python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py --destination_dir ${destination_dir} --varnames T2 MSL gph500
--
GitLab