From e494cc647d8332be0cbdaba9d1643790c41e4abd Mon Sep 17 00:00:00 2001
From: Michael <m.langguth@fz-juelich.de>
Date: Sat, 13 Feb 2021 20:46:56 +0100
Subject: [PATCH] Generalize runscripts for preprocessing step 2.
---
.../preprocess_data_era5_step2_template.sh | 11 ++++++-----
.../preprocess_data_moving_mnist_template.sh | 12 ++++++------
2 files changed, 12 insertions(+), 11 deletions(-)
diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh
index 04ecea9a..a3e397af 100644
--- a/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh
+++ b/video_prediction_tools/HPC_scripts/preprocess_data_era5_step2_template.sh
@@ -6,10 +6,10 @@
#SBATCH --cpus-per-task=1
#SBATCH --output=DataPreprocess_era5_step2-out.%j
#SBATCH --error=DataPreprocess_era5_step2-err.%j
-#SBATCH --time=00:20:00
-#SBATCH --partition=devel
+#SBATCH --time=04:00:00
+#SBATCH --partition=batch
#SBATCH --mail-type=ALL
-#SBATCH --mail-user=b.gong@fz-juelich.de
+#SBATCH --mail-user=me@somewhere.com
######### Template identifier (don't remove) #########
echo "Do not run the template scripts"
@@ -33,8 +33,9 @@ if [ -z ${VIRTUAL_ENV} ]; then
fi
# declare directory-variables which will be modified by config_runscript.py
-base_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/
+base_dir=/my/path/to/base/dir/
sequence_length=20
sequences_per_file=10
# run Preprocessing (step 2 where Tf-records are generated)
-srun python ../main_scripts/main_preprocess_data_step2.py -base_dir ${base_dir} -sequence_length ${sequence_length} -sequences_per_file ${sequences_per_file}
+srun python ../main_scripts/main_preprocess_data_step2.py -base_dir ${base_dir} -sequence_length ${sequence_length} \
+ -sequences_per_file ${sequences_per_file}
diff --git a/video_prediction_tools/HPC_scripts/preprocess_data_moving_mnist_template.sh b/video_prediction_tools/HPC_scripts/preprocess_data_moving_mnist_template.sh
index fadb8b41..cc63f0dd 100644
--- a/video_prediction_tools/HPC_scripts/preprocess_data_moving_mnist_template.sh
+++ b/video_prediction_tools/HPC_scripts/preprocess_data_moving_mnist_template.sh
@@ -6,10 +6,10 @@
#SBATCH --cpus-per-task=1
#SBATCH --output=DataPreprocess_moving_mnist-out.%j
#SBATCH --error=DataPreprocess_moving_mnist-err.%j
-#SBATCH --time=00:20:00
-#SBATCH --partition=devel
+#SBATCH --time=04:00:00
+#SBATCH --partition=batch
#SBATCH --mail-type=ALL
-#SBATCH --mail-user=b.gong@fz-juelich.de
+#SBATCH --mail-user=me@somewhere.com
######### Template identifier (don't remove) #########
echo "Do not run the template scripts"
@@ -32,9 +32,9 @@ if [ -z ${VIRTUAL_ENV} ]; then
fi
fi
-# declare directory-variables which will be modified appropriately during Preprocessing (invoked by mpi_split_data_multi_years.py)
-source_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/moving_mnist
-destination_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/moving_mnist
+# declare directory-variables which will be modified generate_runscript.py
+source_dir=/my/path/to/mnist/raw/data/
+destination_dir=/my/path/to/mnist/tfrecords/
# run Preprocessing (step 2 where Tf-records are generated)
srun python ../video_prediction/datasets/moving_mnist.py ${source_dir} ${destination_dir}
--
GitLab