diff --git a/video_prediction_tools/no_HPC_scripts/data_extraction_era5_template.sh b/video_prediction_tools/no_HPC_scripts/data_extraction_era5_template.sh index 4e3c4d3a96f4ffbe1f0c238dc650eba7760151ff..3d3c2f3dbc54d294f392943a416cf3eef79be679 100644 --- a/video_prediction_tools/no_HPC_scripts/data_extraction_era5_template.sh +++ b/video_prediction_tools/no_HPC_scripts/data_extraction_era5_template.sh @@ -16,16 +16,16 @@ source_dir=/my/path/to/era5 destination_dir=/my/path/to/extracted/data varmap_file=/my/path/to/varmapping/file -years=( "2007" ) +years=$(seq 2007 2020) #The number of nodes should be equal to the number of 1 preprocessed folder plus 1 n_nodes=3 # Run data extraction -for year in "${years[@]}"; do +for year in ${years}; do echo "Perform ERA5-data extraction for year ${year}" python ../main_scripts/main_data_extraction.py --source_dir ${source_dir} --target_dir ${destination_dir} \ - --year ${year} --varslist_path ${varmap_file} + --year "${year}" --varslist_path ${varmap_file} done diff --git a/video_prediction_tools/no_HPC_scripts/preprocess_data_era5_step1_template.sh b/video_prediction_tools/no_HPC_scripts/preprocess_data_era5_step1_template.sh index 8519569a32d6a86c98363c85eb707ffa75e4960f..35e9f87d75e28903c928814c7694a1165f8e8b01 100644 --- a/video_prediction_tools/no_HPC_scripts/preprocess_data_era5_step1_template.sh +++ b/video_prediction_tools/no_HPC_scripts/preprocess_data_era5_step1_template.sh @@ -19,10 +19,10 @@ if [ -z ${VIRTUAL_ENV} ]; then fi #select years and variables for dataset and define target domain -years=( "2007" ) -variables=( "2t" ) -sw_corner=( 10 20) -nyx=( 40 40 ) +years=$(seq 2007 2020) +variables=( "t2" "t_850" "tcc" ) +sw_corner=( 38.4 0.0 ) +nyx=( 56 92 ) #your source dir and target dir source_dir=/home/b.gong/data_era5 diff --git a/video_prediction_tools/no_HPC_scripts/preprocess_data_era5_step2_template.sh b/video_prediction_tools/no_HPC_scripts/preprocess_data_era5_step2_template.sh index 441924e7238cf87edbcca4b5a757f60358c3ecc4..4d7356f6ccddc13020d18a5193721463010b8807 100644 --- a/video_prediction_tools/no_HPC_scripts/preprocess_data_era5_step2_template.sh +++ b/video_prediction_tools/no_HPC_scripts/preprocess_data_era5_step2_template.sh @@ -15,7 +15,7 @@ VIRT_ENV_NAME=venv_test source_dir=/my/path/to/pkl/files/ destination_dir=/my/path/to/tfrecords/files -sequence_length=20 +sequence_length=24 sequences_per_file=10 #the number of the nodes should be the number of processed folder (month) plus 1