diff --git a/video_prediction_tools/other_scripts/preprocess_data_era5_step2_template.sh b/video_prediction_tools/other_scripts/preprocess_data_era5_step2_template.sh index 2d452c568cc53f776082e87019e70e1846db96a3..9af83c2d845ddb3397d9ee76145446e8136bc05e 100644 --- a/video_prediction_tools/other_scripts/preprocess_data_era5_step2_template.sh +++ b/video_prediction_tools/other_scripts/preprocess_data_era5_step2_template.sh @@ -8,11 +8,11 @@ source ../virtual_envs/${VIRT_ENV_NAME}/bin/activate sequence_length=20 sequences_per_file=10 - -source_dir=/home/b.gong/preprocessed_data-24x24-990N2010E-var167/pickle +source_dir=/home/b.gong/preprocessed_data-40x40-990N2010E-2t/pickle base_dir="$(dirname "$source_dir")" destination_dir=${base_dir}/tfrecords -n_nodes=2 +#the number of the nodes should be the number of processed folder (month) plus 1 +n_nodes=3 mpirun -n ${n_nodes} python3 ../main_scripts/main_preprocess_data_step2.py -source_dir ${source_dir} -dest_dir ${destination_dir} \ -sequence_length ${sequence_length} -sequences_per_file ${sequences_per_file}