diff --git a/video_prediction_savp/HPC_scripts/data_extraction_era5_template.sh b/video_prediction_savp/HPC_scripts/data_extraction_era5_template.sh index a80a3b8779908fc51121c6682817f20ec197a327..68e1c97c2575e6002694ca895f618a424f22d822 100644 --- a/video_prediction_savp/HPC_scripts/data_extraction_era5_template.sh +++ b/video_prediction_savp/HPC_scripts/data_extraction_era5_template.sh @@ -5,9 +5,9 @@ #SBATCH --ntasks=13 ##SBATCH --ntasks-per-node=13 #SBATCH --cpus-per-task=1 -#SBATCH --output=DataExtraction-out.%j -#SBATCH --error=DataExtraction-err.%j -#SBATCH --time=05:00:00 +#SBATCH --output=data_extraction_era5-out.%j +#SBATCH --error=data_extraction_era5-err.%j +#SBATCH --time=00:20:00 #SBATCH --partition=devel #SBATCH --mail-type=ALL #SBATCH --mail-user=b.gong@fz-juelich.de diff --git a/video_prediction_savp/env_setup/create_env.sh b/video_prediction_savp/env_setup/create_env.sh index 9f8a4c5aa007695a6d668040aacf08e158b3a12f..d014f5d37bd9f8f8eb900ff1529d98445b69f53d 100755 --- a/video_prediction_savp/env_setup/create_env.sh +++ b/video_prediction_savp/env_setup/create_env.sh @@ -32,7 +32,7 @@ fi # list of (Batch) scripts used for the steps in the workflow # !!! Expects that a template named [script_name]_template.sh exists!!! -workflow_scripts=(DataExtraction DataPreprocess DataPreprocess2tf train_era5 generate_era5 DatePreprocess2tf_movingmnist train_movingmnist generate_movingmnist) +workflow_scripts=(data_extraction_era5 preprocess_data_era5_step1 preprocess_data_era5_step2 train_model_era5 visualize_postprocess_era5 preprocess_data_moving_mnist train_model_moving_mnist visualize_postprocess_moving_mnist) HOST_NAME=`hostname` ENV_NAME=$1 diff --git a/video_prediction_savp/video_prediction/datasets/prepare_era5_data.py b/video_prediction_savp/utils/prepare_era5_data.py similarity index 100% rename from video_prediction_savp/video_prediction/datasets/prepare_era5_data.py rename to video_prediction_savp/utils/prepare_era5_data.py diff --git a/video_prediction_savp/video_prediction/datasets/process_netCDF_v2.py b/video_prediction_savp/utils/process_netCDF_v2.py similarity index 100% rename from video_prediction_savp/video_prediction/datasets/process_netCDF_v2.py rename to video_prediction_savp/utils/process_netCDF_v2.py diff --git a/video_prediction_savp/video_prediction/datasets/__init__.py b/video_prediction_savp/video_prediction/datasets/__init__.py index f58607c2f4c14047aefb36956e98bd228a30aeb1..2cdad80fc658f4c1e4aa4725ac0d2da0cb9a65bd 100644 --- a/video_prediction_savp/video_prediction/datasets/__init__.py +++ b/video_prediction_savp/video_prediction/datasets/__init__.py @@ -6,7 +6,7 @@ from .softmotion_dataset import SoftmotionVideoDataset from .kth_dataset import KTHVideoDataset from .ucf101_dataset import UCF101VideoDataset from .cartgripper_dataset import CartgripperVideoDataset -from .era5_dataset_v2 import ERA5Dataset_v2 +from .era5_dataset import ERA5Dataset from .moving_mnist import MovingMnist #from .era5_dataset_v2_anomaly import ERA5Dataset_v2_anomaly diff --git a/video_prediction_savp/video_prediction/datasets/era5_dataset.py b/video_prediction_savp/video_prediction/datasets/era5_dataset.py index 8835363a002c06f7e5cfcf337e4db07d280a3bc6..4d80aacf701f74aa03a948765086905b386c6bb5 100644 --- a/video_prediction_savp/video_prediction/datasets/era5_dataset.py +++ b/video_prediction_savp/video_prediction/datasets/era5_dataset.py @@ -12,8 +12,7 @@ from video_prediction.datasets.base_dataset import VarLenFeatureVideoDataset # ML 2020/04/14: hack for getting functions of process_netCDF_v2: from os import path import sys -sys.path.append(path.abspath('../../workflow_parallel_frame_prediction/')) -import DataPreprocess.process_netCDF_v2 +import video_prediction.datasets.process_netCDF_v2 from general_utils import get_unique_vars from statistics import Calc_data_stat from metadata import MetaData @@ -26,9 +25,9 @@ import glob -class ERA5Dataset_v2(VarLenFeatureVideoDataset): +class ERA5Dataset(VarLenFeatureVideoDataset): def __init__(self, *args, **kwargs): - super(ERA5Dataset_v2, self).__init__(*args, **kwargs) + super(ERA5Dataset, self).__init__(*args, **kwargs) from google.protobuf.json_format import MessageToDict example = next(tf.python_io.tf_record_iterator(self.filenames[0])) dict_message = MessageToDict(tf.train.Example.FromString(example)) @@ -39,7 +38,7 @@ class ERA5Dataset_v2(VarLenFeatureVideoDataset): self.state_like_names_and_shapes['images'] = 'images/encoded', self.image_shape def get_default_hparams_dict(self): - default_hparams = super(ERA5Dataset_v2, self).get_default_hparams_dict() + default_hparams = super(ERA5Dataset, self).get_default_hparams_dict() hparams = dict( context_frames=10,#Bing: Todo oriignal is 10 sequence_length=20,#bing: TODO original is 20, @@ -122,6 +121,9 @@ class ERA5Dataset_v2(VarLenFeatureVideoDataset): # _parser, batch_size, drop_remainder=True, num_parallel_calls=num_parallel_calls)) # Bing: Parallel data mapping, num_parallel_calls normally depends on the hardware, however, normally should be equal to be the usalbe number of CPUs dataset = dataset.prefetch(batch_size) # Bing: Take the data to buffer inorder to save the waiting time for GPU return dataset +/V2 + +