diff --git a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh index d4741e796c2762d8056204bc36e21655af1e70dd..f2a3415cc01ef993b298deef447a368ed1b4bee5 100644 --- a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh +++ b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh @@ -39,11 +39,11 @@ source_dir=/my/path/to/era5 destination_dir=/my/path/to/extracted/data varmap_file=/my/path/to/varmapping/file -years=( "2015" ) +years=$(seq 2007 2020) # Run data extraction -for year in "${years[@]}"; do +for year in ${years}; do echo "Perform ERA5-data extraction for year ${year}" srun python ../main_scripts/main_data_extraction.py --source_dir ${source_dir} --target_dir ${destination_dir} \ - --year ${year} --varslist_path ${varmap_file} + --year "${year}" --varslist_path ${varmap_file} done diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh index 990095c1f5fa00e4058e08d355830e0fa620b0f3..3e48cbadf48f65f9c1753a76590d47baff059b14 100644 --- a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh +++ b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step1_template.sh @@ -36,10 +36,10 @@ source ../env_setup/modules_preprocess+extract.sh # select years and variables for dataset and define target domain -years=( "2015" ) -variables=( "t2" "t2" "t2" ) -sw_corner=( -999.9 -999.9) -nyx=( -999 -999 ) +years=$(seq 2007 2020) +variables=( "t2" "t_850" "tcc" ) +sw_corner=( 38.4 0.0 ) +nyx=( 56 92 ) # set some paths # note, that destination_dir is adjusted during runtime based on the data @@ -47,7 +47,7 @@ source_dir=/my/path/to/extracted/data/ destination_dir=/my/path/to/pickle/files # execute Python-scripts -for year in "${years[@]}"; do +for year in ${years}; do echo "start preprocessing data for year ${year}" srun python ../main_scripts/main_preprocess_data_step1.py \ --source_dir ${source_dir} --destination_dir ${destination_dir} --years "${year}" \ diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh index daa48d352ce6b1eca9c2f76692e68ca3e786273e..0c01282c5d83eb59a8bb0744165bbaddcbd4d72d 100644 --- a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh +++ b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh @@ -46,7 +46,7 @@ module purge source_dir=/my/path/to/pkl/files/ destination_dir=/my/path/to/tfrecords/files -sequence_length=20 +sequence_length=24 sequences_per_file=10 # run Preprocessing (step 2 where Tf-records are generated) export CUDA_VISIBLE_DEVICES=0 diff --git a/video_prediction_tools/data_preprocess/era5_varmapping_template.json b/video_prediction_tools/data_preprocess/era5_varmapping_template.json index d4aa1d16a18c50b37b42a3a8cd89c8da80214155..96f422091bb708f4ca4c01a473cc82d722283296 100644 --- a/video_prediction_tools/data_preprocess/era5_varmapping_template.json +++ b/video_prediction_tools/data_preprocess/era5_varmapping_template.json @@ -9,7 +9,7 @@ # The value of the 'pl'-key denotes the pressure level (in Pa) onto which the data is interpolated # !!! This file should be only adapted if you are familiar with the ERA5 grib files!!! { -"surface": ["2t", "tcc","msl","10u","10v"], +"surface": ["2t", "tcc", "msl","10u","10v"], "multi":{ "t" : {