From ec70acf086db39bf9ef6bb5f84eac407a5b926fc Mon Sep 17 00:00:00 2001
From: Michael <m.langguth@fz-juelich.de>
Date: Tue, 19 Jul 2022 08:55:43 +0200
Subject: [PATCH] Adapt runscripts under HPC_scripts so that the template carry
defautl values (for ease user application).
---
.../HPC_scripts/data_extraction_era5_template.sh | 6 +++---
.../HPC_scripts/preprocess_data_era5_step1_template.sh | 10 +++++-----
.../HPC_scripts/preprocess_data_era5_step2_template.sh | 2 +-
.../data_preprocess/era5_varmapping_template.json | 2 +-
4 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh
index d4741e79..f2a3415c 100644
--- a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh
+++ b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh
@@ -39,11 +39,11 @@ source_dir=/my/path/to/era5
destination_dir=/my/path/to/extracted/data
varmap_file=/my/path/to/varmapping/file
-years=( "2015" )
+years=$(seq 2007 2020)
# Run data extraction
-for year in "${years[@]}"; do
+for year in ${years}; do
echo "Perform ERA5-data extraction for year ${year}"
srun python ../main_scripts/main_data_extraction.py --source_dir ${source_dir} --target_dir ${destination_dir} \
- --year ${year} --varslist_path ${varmap_file}
+ --year "${year}" --varslist_path ${varmap_file}
done
diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh
index 990095c1..3e48cbad 100644
--- a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh
+++ b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh
@@ -36,10 +36,10 @@ source ../env_setup/modules_preprocess+extract.sh
# select years and variables for dataset and define target domain
-years=( "2015" )
-variables=( "t2" "t2" "t2" )
-sw_corner=( -999.9 -999.9)
-nyx=( -999 -999 )
+years=$(seq 2007 2020)
+variables=( "t2" "t_850" "tcc" )
+sw_corner=( 38.4 0.0 )
+nyx=( 56 92 )
# set some paths
# note, that destination_dir is adjusted during runtime based on the data
@@ -47,7 +47,7 @@ source_dir=/my/path/to/extracted/data/
destination_dir=/my/path/to/pickle/files
# execute Python-scripts
-for year in "${years[@]}"; do
+for year in ${years}; do
echo "start preprocessing data for year ${year}"
srun python ../main_scripts/main_preprocess_data_step1.py \
--source_dir ${source_dir} --destination_dir ${destination_dir} --years "${year}" \
diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh
index daa48d35..0c01282c 100644
--- a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh
+++ b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh
@@ -46,7 +46,7 @@ module purge
source_dir=/my/path/to/pkl/files/
destination_dir=/my/path/to/tfrecords/files
-sequence_length=20
+sequence_length=24
sequences_per_file=10
# run Preprocessing (step 2 where Tf-records are generated)
export CUDA_VISIBLE_DEVICES=0
diff --git a/video_prediction_tools/data_preprocess/era5_varmapping_template.json b/video_prediction_tools/data_preprocess/era5_varmapping_template.json
index d4aa1d16..96f42209 100644
--- a/video_prediction_tools/data_preprocess/era5_varmapping_template.json
+++ b/video_prediction_tools/data_preprocess/era5_varmapping_template.json
@@ -9,7 +9,7 @@
# The value of the 'pl'-key denotes the pressure level (in Pa) onto which the data is interpolated
# !!! This file should be only adapted if you are familiar with the ERA5 grib files!!!
{
-"surface": ["2t", "tcc","msl","10u","10v"],
+"surface": ["2t", "tcc", "msl","10u","10v"],
"multi":{
"t" : {
--
GitLab