From 4b14784bd4d0f068ff8af32347c7208b8bfd2c46 Mon Sep 17 00:00:00 2001
From: Bing Gong <gongbing1112@gmail.com>
Date: Fri, 20 Mar 2020 09:26:46 +0100
Subject: [PATCH] add DataExtraction submission job

---
 HPC_scripts/DataExtraction.sh | 25 +++++++++++++++++++++++++
 HPC_scripts/generate_era5.sh  |  1 -
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100755 HPC_scripts/DataExtraction.sh

diff --git a/HPC_scripts/DataExtraction.sh b/HPC_scripts/DataExtraction.sh
new file mode 100755
index 00000000..dbbc35bd
--- /dev/null
+++ b/HPC_scripts/DataExtraction.sh
@@ -0,0 +1,25 @@
+
+#!/bin/bash -x
+#SBATCH --account=deepacf
+#SBATCH --nodes=1
+#SBATCH --ntasks=13
+##SBATCH --ntasks-per-node=13
+#SBATCH --cpus-per-task=1
+#SBATCH --output=pystager-out.%j
+#SBATCH --error=pystager-err.%j
+#SBATCH --time=00:20:00
+#SBATCH --partition=devel
+#SBATCH --mail-type=ALL
+#SBATCH --mail-user=b.gong@fz-juelich.de
+##jutil env activate -p deepacf
+
+module --force purge
+module use $OTHERSTAGES
+module load Stages/2019a
+module addad Intel/2019.3.199-GCC-8.3.0  ParaStationMPI/5.2.2-1
+module load h5py/2.9.0-Python-3.6.8
+module load mpi4py/3.0.1-Python-3.6.8
+
+#module load mpi4py/3.0.1-Python-3.6.8
+
+srun python mpi_stager_v2.py --source_dir /p/fastdata/slmet/slmet111/met_data/ecmwf/era5/nc/2017/ --destination_dir /p/scratch/deepacf/bing/extractedData
diff --git a/HPC_scripts/generate_era5.sh b/HPC_scripts/generate_era5.sh
index e0481b47..dc7377e5 100755
--- a/HPC_scripts/generate_era5.sh
+++ b/HPC_scripts/generate_era5.sh
@@ -20,7 +20,6 @@ module load ParaStationMPI/5.2.2-1
 module load TensorFlow/1.13.1-GPU-Python-3.6.8
 module load netcdf4-python/1.5.0.1-Python-3.6.8
 module load h5py/2.9.0-Python-3.6.8
-source mandarine/bin/activate
 
 python scripts/generate_transfer_learning_finetune.py --input_dir data/era5_size_64_64_3_norm_dup --dataset_hparams sequence_length=20 --checkpoint logs/era5_64_64_3_norm_2016/ours_savp --mode test --results_dir results_test_samples/era5_size_64_64_3_norm_2016  --batch_size 4 --dataset era5 
 #srun  python scripts/train.py --input_dir data/era5 --dataset era5  --model savp --model_hparams_dict hparams/kth/ours_savp/model_hparams.json --output_dir logs/era5/ours_savp
-- 
GitLab