diff --git a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh index dd64758350c1944530d0f608295d3e03dd40e677..a59585b6388dd66f324337ea39c571dc61b90c00 100644 --- a/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh +++ b/video_prediction_tools/HPC_scripts/data_extraction_era5_template.sh @@ -7,7 +7,7 @@ #SBATCH --cpus-per-task=1 #SBATCH --output=data_extraction_era5-out.%j #SBATCH --error=data_extraction_era5-err.%j -#SBATCH --time=00:20:00 +#SBATCH --time=04:20:00 #SBATCH --partition=devel #SBATCH --mail-type=ALL #SBATCH --mail-user=b.gong@fz-juelich.de diff --git a/video_prediction_tools/HPC_scripts/train_model_era5_template.sh b/video_prediction_tools/HPC_scripts/train_model_era5_template.sh index aa52ce1b85a1a16861ca5bebc83e5325806a5ce3..a50420c38d6d992e18c580d6f29b6ee071aa0f4b 100644 --- a/video_prediction_tools/HPC_scripts/train_model_era5_template.sh +++ b/video_prediction_tools/HPC_scripts/train_model_era5_template.sh @@ -45,6 +45,9 @@ datasplit_dict=../data_split/cv_test.json model_hparams=${destination_dir}/model_hparams.json dataset=era5 +#If you train savp, Please uncomment the following CUDA configuration +#CUDA_VISIBLE_DEVICES=1 + # run training srun python ../main_scripts/main_train_models.py --input_dir ${source_dir} --datasplit_dict ${datasplit_dict} --dataset ${dataset} --model ${model} --model_hparams_dict ${model_hparams} --output_dir ${destination_dir} diff --git a/video_prediction_tools/data_preprocess/prepare_era5_data.py b/video_prediction_tools/data_preprocess/prepare_era5_data.py index 438a0c2b58ae3a6d7cbd4aa8a1cb67e10577febc..a6d19953ce3cd6983384c6cdfe9d4478a66b4137 100644 --- a/video_prediction_tools/data_preprocess/prepare_era5_data.py +++ b/video_prediction_tools/data_preprocess/prepare_era5_data.py @@ -150,7 +150,7 @@ def process_era5_in_dir(job_name,src_dir,target_dir): #create a subdirectory based on months target_dir2 = os.path.join(target_dir,job_name) print("The processed files are going to be saved to directory {}".format(target_dir2)) - if not os.path.exists(target_dir2): os.mkdir(target_dir2) + if not os.path.exists(target_dir2): os.makedirs(target_dir2, exist_ok=True) for src_file in files: if src_file.endswith(".nc"): if os.path.exists(os.path.join(target_dir2, src_file)): diff --git a/video_prediction_tools/main_scripts/main_data_extraction.py b/video_prediction_tools/main_scripts/main_data_extraction.py index 0813e9b6a06e2733ad112d5fc31778ae966ae850..25c7e492550c9e7dfca988733857debb55e47d06 100644 --- a/video_prediction_tools/main_scripts/main_data_extraction.py +++ b/video_prediction_tools/main_scripts/main_data_extraction.py @@ -12,8 +12,6 @@ import logging import time from utils.external_function import directory_scanner from utils.external_function import load_distributor -from utils.external_function import hash_directory -from utils.external_function import md5 from data_preprocess.prepare_era5_data import * # How to Run it! # mpirun -np 6 python mpi_stager_v2.py diff --git a/video_prediction_tools/main_scripts/main_train_models.py b/video_prediction_tools/main_scripts/main_train_models.py index 94b9475f7c19fcd7364d5427ffa8ad17623b0b63..14f607e64a3a36ff8606be53711e18a82eab95ad 100644 --- a/video_prediction_tools/main_scripts/main_train_models.py +++ b/video_prediction_tools/main_scripts/main_train_models.py @@ -46,7 +46,7 @@ class TrainModel(object): save_interval : int, how many steps for saving the train/val loss be saved """ self.input_dir = os.path.normpath(input_dir) - self.output_dir = output_dir + self.output_dir = os.path.normpath(output_dir) self.datasplit_dict = datasplit_dict self.model_hparams_dict = model_hparams_dict self.checkpoint = checkpoint diff --git a/video_prediction_tools/main_scripts/main_visualize_postprocess.py b/video_prediction_tools/main_scripts/main_visualize_postprocess.py index c78eecf87fa704bf06c700825910914f63110094..7b627ab6db3cd8534823e6c0b2e36f489368510c 100644 --- a/video_prediction_tools/main_scripts/main_visualize_postprocess.py +++ b/video_prediction_tools/main_scripts/main_visualize_postprocess.py @@ -49,8 +49,8 @@ class Postprocess(TrainModel,ERA5Pkl2Tfrecords): seed :seed for control test samples """ - self.input_dir = input_dir - self.results_dir = self.output_dir = results_dir + self.input_dir = os.path.normpath(input_dir) + self.results_dir = self.output_dir = os.path.normpath(results_dir) if not os.path.exists(self.results_dir):os.makedirs(self.results_dir) self.batch_size = batch_size self.gpu_mem_frac = gpu_mem_frac