From c80c4b649b1754265bb56bbd2c05a57ae4a7845a Mon Sep 17 00:00:00 2001
From: gong1 <b.gong@fz-juelich.de>
Date: Tue, 22 Sep 2020 14:25:08 +0200
Subject: [PATCH] change the import module names since the script names changed
---
.../HPC_scripts/data_extraction_era5_template.sh | 6 +++---
video_prediction_savp/env_setup/create_env.sh | 2 +-
.../datasets => utils}/prepare_era5_data.py | 0
.../datasets => utils}/process_netCDF_v2.py | 0
.../video_prediction/datasets/__init__.py | 2 +-
.../video_prediction/datasets/era5_dataset.py | 12 +++++++-----
6 files changed, 12 insertions(+), 10 deletions(-)
rename video_prediction_savp/{video_prediction/datasets => utils}/prepare_era5_data.py (100%)
rename video_prediction_savp/{video_prediction/datasets => utils}/process_netCDF_v2.py (100%)
diff --git a/video_prediction_savp/HPC_scripts/data_extraction_era5_template.sh b/video_prediction_savp/HPC_scripts/data_extraction_era5_template.sh
index a80a3b87..68e1c97c 100644
--- a/video_prediction_savp/HPC_scripts/data_extraction_era5_template.sh
+++ b/video_prediction_savp/HPC_scripts/data_extraction_era5_template.sh
@@ -5,9 +5,9 @@
#SBATCH --ntasks=13
##SBATCH --ntasks-per-node=13
#SBATCH --cpus-per-task=1
-#SBATCH --output=DataExtraction-out.%j
-#SBATCH --error=DataExtraction-err.%j
-#SBATCH --time=05:00:00
+#SBATCH --output=data_extraction_era5-out.%j
+#SBATCH --error=data_extraction_era5-err.%j
+#SBATCH --time=00:20:00
#SBATCH --partition=devel
#SBATCH --mail-type=ALL
#SBATCH --mail-user=b.gong@fz-juelich.de
diff --git a/video_prediction_savp/env_setup/create_env.sh b/video_prediction_savp/env_setup/create_env.sh
index 9f8a4c5a..d014f5d3 100755
--- a/video_prediction_savp/env_setup/create_env.sh
+++ b/video_prediction_savp/env_setup/create_env.sh
@@ -32,7 +32,7 @@ fi
# list of (Batch) scripts used for the steps in the workflow
# !!! Expects that a template named [script_name]_template.sh exists!!!
-workflow_scripts=(DataExtraction DataPreprocess DataPreprocess2tf train_era5 generate_era5 DatePreprocess2tf_movingmnist train_movingmnist generate_movingmnist)
+workflow_scripts=(data_extraction_era5 preprocess_data_era5_step1 preprocess_data_era5_step2 train_model_era5 visualize_postprocess_era5 preprocess_data_moving_mnist train_model_moving_mnist visualize_postprocess_moving_mnist)
HOST_NAME=`hostname`
ENV_NAME=$1
diff --git a/video_prediction_savp/video_prediction/datasets/prepare_era5_data.py b/video_prediction_savp/utils/prepare_era5_data.py
similarity index 100%
rename from video_prediction_savp/video_prediction/datasets/prepare_era5_data.py
rename to video_prediction_savp/utils/prepare_era5_data.py
diff --git a/video_prediction_savp/video_prediction/datasets/process_netCDF_v2.py b/video_prediction_savp/utils/process_netCDF_v2.py
similarity index 100%
rename from video_prediction_savp/video_prediction/datasets/process_netCDF_v2.py
rename to video_prediction_savp/utils/process_netCDF_v2.py
diff --git a/video_prediction_savp/video_prediction/datasets/__init__.py b/video_prediction_savp/video_prediction/datasets/__init__.py
index f58607c2..2cdad80f 100644
--- a/video_prediction_savp/video_prediction/datasets/__init__.py
+++ b/video_prediction_savp/video_prediction/datasets/__init__.py
@@ -6,7 +6,7 @@ from .softmotion_dataset import SoftmotionVideoDataset
from .kth_dataset import KTHVideoDataset
from .ucf101_dataset import UCF101VideoDataset
from .cartgripper_dataset import CartgripperVideoDataset
-from .era5_dataset_v2 import ERA5Dataset_v2
+from .era5_dataset import ERA5Dataset
from .moving_mnist import MovingMnist
#from .era5_dataset_v2_anomaly import ERA5Dataset_v2_anomaly
diff --git a/video_prediction_savp/video_prediction/datasets/era5_dataset.py b/video_prediction_savp/video_prediction/datasets/era5_dataset.py
index 8835363a..4d80aacf 100644
--- a/video_prediction_savp/video_prediction/datasets/era5_dataset.py
+++ b/video_prediction_savp/video_prediction/datasets/era5_dataset.py
@@ -12,8 +12,7 @@ from video_prediction.datasets.base_dataset import VarLenFeatureVideoDataset
# ML 2020/04/14: hack for getting functions of process_netCDF_v2:
from os import path
import sys
-sys.path.append(path.abspath('../../workflow_parallel_frame_prediction/'))
-import DataPreprocess.process_netCDF_v2
+import video_prediction.datasets.process_netCDF_v2
from general_utils import get_unique_vars
from statistics import Calc_data_stat
from metadata import MetaData
@@ -26,9 +25,9 @@ import glob
-class ERA5Dataset_v2(VarLenFeatureVideoDataset):
+class ERA5Dataset(VarLenFeatureVideoDataset):
def __init__(self, *args, **kwargs):
- super(ERA5Dataset_v2, self).__init__(*args, **kwargs)
+ super(ERA5Dataset, self).__init__(*args, **kwargs)
from google.protobuf.json_format import MessageToDict
example = next(tf.python_io.tf_record_iterator(self.filenames[0]))
dict_message = MessageToDict(tf.train.Example.FromString(example))
@@ -39,7 +38,7 @@ class ERA5Dataset_v2(VarLenFeatureVideoDataset):
self.state_like_names_and_shapes['images'] = 'images/encoded', self.image_shape
def get_default_hparams_dict(self):
- default_hparams = super(ERA5Dataset_v2, self).get_default_hparams_dict()
+ default_hparams = super(ERA5Dataset, self).get_default_hparams_dict()
hparams = dict(
context_frames=10,#Bing: Todo oriignal is 10
sequence_length=20,#bing: TODO original is 20,
@@ -122,6 +121,9 @@ class ERA5Dataset_v2(VarLenFeatureVideoDataset):
# _parser, batch_size, drop_remainder=True, num_parallel_calls=num_parallel_calls)) # Bing: Parallel data mapping, num_parallel_calls normally depends on the hardware, however, normally should be equal to be the usalbe number of CPUs
dataset = dataset.prefetch(batch_size) # Bing: Take the data to buffer inorder to save the waiting time for GPU
return dataset
+/V2
+
+
--
GitLab