diff --git a/HPC_scripts/DataExtraction.sh b/HPC_scripts/DataExtraction.sh new file mode 100755 index 0000000000000000000000000000000000000000..dbbc35bd983b91d87bc63154b169dbc5219a7955 --- /dev/null +++ b/HPC_scripts/DataExtraction.sh @@ -0,0 +1,25 @@ + +#!/bin/bash -x +#SBATCH --account=deepacf +#SBATCH --nodes=1 +#SBATCH --ntasks=13 +##SBATCH --ntasks-per-node=13 +#SBATCH --cpus-per-task=1 +#SBATCH --output=pystager-out.%j +#SBATCH --error=pystager-err.%j +#SBATCH --time=00:20:00 +#SBATCH --partition=devel +#SBATCH --mail-type=ALL +#SBATCH --mail-user=b.gong@fz-juelich.de +##jutil env activate -p deepacf + +module --force purge +module use $OTHERSTAGES +module load Stages/2019a +module addad Intel/2019.3.199-GCC-8.3.0 ParaStationMPI/5.2.2-1 +module load h5py/2.9.0-Python-3.6.8 +module load mpi4py/3.0.1-Python-3.6.8 + +#module load mpi4py/3.0.1-Python-3.6.8 + +srun python mpi_stager_v2.py --source_dir /p/fastdata/slmet/slmet111/met_data/ecmwf/era5/nc/2017/ --destination_dir /p/scratch/deepacf/bing/extractedData diff --git a/HPC_scripts/generate_era5.sh b/HPC_scripts/generate_era5.sh index e0481b47d1bc9e3b02276a916bf762601373aa98..dc7377e5103a44c34e2de599f24e8ffee60629f2 100755 --- a/HPC_scripts/generate_era5.sh +++ b/HPC_scripts/generate_era5.sh @@ -20,7 +20,6 @@ module load ParaStationMPI/5.2.2-1 module load TensorFlow/1.13.1-GPU-Python-3.6.8 module load netcdf4-python/1.5.0.1-Python-3.6.8 module load h5py/2.9.0-Python-3.6.8 -source mandarine/bin/activate python scripts/generate_transfer_learning_finetune.py --input_dir data/era5_size_64_64_3_norm_dup --dataset_hparams sequence_length=20 --checkpoint logs/era5_64_64_3_norm_2016/ours_savp --mode test --results_dir results_test_samples/era5_size_64_64_3_norm_2016 --batch_size 4 --dataset era5 #srun python scripts/train.py --input_dir data/era5 --dataset era5 --model savp --model_hparams_dict hparams/kth/ours_savp/model_hparams.json --output_dir logs/era5/ours_savp