diff --git a/video_prediction_savp/HPC_scripts/generate_movingmnist.sh b/video_prediction_savp/HPC_scripts/generate_movingmnist.sh index f6a36d1a366a1e1492546e8a8ac14760cc434098..1de81d2543d255a160ff811ff391a963ef712bde 100755 --- a/video_prediction_savp/HPC_scripts/generate_movingmnist.sh +++ b/video_prediction_savp/HPC_scripts/generate_movingmnist.sh @@ -10,7 +10,7 @@ #SBATCH --gres=gpu:1 #SBATCH --partition=develgpus #SBATCH --mail-type=ALL -#SBATCH --mail-user=b.gong@fz-juelich.de +#SBATCH --mail-user=s.stadtler@fz-juelich.de ##jutil env activate -p cjjsc42 # Name of virtual environment diff --git a/video_prediction_savp/HPC_scripts/train_movingmnist.sh b/video_prediction_savp/HPC_scripts/train_movingmnist.sh index f62d333dbf01db0affbf72a3e1ef1ecd96b94ec7..85959e52d148f2120c77e5543a79147d50427838 100755 --- a/video_prediction_savp/HPC_scripts/train_movingmnist.sh +++ b/video_prediction_savp/HPC_scripts/train_movingmnist.sh @@ -4,8 +4,8 @@ #SBATCH --ntasks=1 ##SBATCH --ntasks-per-node=1 #SBATCH --cpus-per-task=1 -#SBATCH --output=train_era5-out.%j -#SBATCH --error=train_era5-err.%j +#SBATCH --output=train_moving_mnist-out.%j +#SBATCH --error=train_moving_mnist-err.%j #SBATCH --time=00:20:00 #SBATCH --gres=gpu:1 #SBATCH --partition=develgpus @@ -36,9 +36,9 @@ fi source_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/preprocessedData/moving_mnist destination_dir=/p/project/deepacf/deeprain/video_prediction_shared_folder/models/moving_mnist -# for choosing the model, convLSTM,savp, mcnet,vae,convLSTM_Loliver +# for choosing the model, convLSTM,savp, mcnet,vae model=convLSTM model_hparams=../hparams/era5/${model}/model_hparams.json # rund training -srun python ../scripts/train_moving_mnist.py --input_dir ${source_dir}/tfrecords/ --dataset moving_mnist --model ${model} --model_hparams_dict ${model_hparams} --output_dir ${destination_dir}/${model}/ --checkpoint ${destination_dir}/${model}/ +srun python ../scripts/train_dummy_moving_mnist.py --input_dir ${source_dir}/tfrecords/ --dataset moving_mnist --model ${model} --model_hparams_dict ${model_hparams} --output_dir ${destination_dir}/${model}/ diff --git a/video_prediction_savp/scripts/train_moving_mnist.py b/video_prediction_savp/scripts/train_moving_mnist.py index 0cb9a6be0d23449dbdf30cc316815e2a33b29de1..fe7d2f065e895b40844337c22c74e6007f183bd4 100644 --- a/video_prediction_savp/scripts/train_moving_mnist.py +++ b/video_prediction_savp/scripts/train_moving_mnist.py @@ -276,8 +276,10 @@ def main(): val_losses=[] run_start_time = time.time() for step in range(start_step,total_steps): - #global_step = sess.run(global_step):q - + #global_step = sess.run(global_step) + # +++ Scarlet 20200813 + timeit_start = time.time() + # --- Scarlet 20200813 print ("step:", step) val_handle_eval = sess.run(val_handle) @@ -342,7 +344,11 @@ def main(): print ("The model name does not exist") #print("saving model to", args.output_dir) - saver.save(sess, os.path.join(args.output_dir, "model"), global_step=step)# + saver.save(sess, os.path.join(args.output_dir, "model"), global_step=step) + # +++ Scarlet 20200813 + timeit_end = time.time() + # --- Scarlet 20200813 + print("time needed for this step", timeit_end - timeit_start, ' s') train_time = time.time() - run_start_time results_dict = {"train_time":train_time, "total_steps":total_steps} @@ -352,6 +358,9 @@ def main(): print("val_losses:",val_losses) plot_train(train_losses,val_losses,args.output_dir) print("Done") + # +++ Scarlet 20200814 + print("Total training time:", train_time/60., "min") + # +++ Scarlet 20200814 if __name__ == '__main__': main() diff --git a/video_prediction_savp/video_prediction/layers/layer_def.py b/video_prediction_savp/video_prediction/layers/layer_def.py index 738e139df0e155ca294fe43edd03a2d79fc1f532..1ceac662136548fde65511815795d184fe91fac1 100644 --- a/video_prediction_savp/video_prediction/layers/layer_def.py +++ b/video_prediction_savp/video_prediction/layers/layer_def.py @@ -74,6 +74,8 @@ def conv_layer(inputs, kernel_size, stride, num_features, idx, initializer=tf.co conv_rect = tf.nn.elu(conv_biased, name = '{0}_conv'.format(idx)) elif activate == "leaky_relu": conv_rect = tf.nn.leaky_relu(conv_biased, name = '{0}_conv'.format(idx)) + elif activate == "sigmoid": + conv_rect = tf.nn.sigmoid(conv_biased, name = '{0}_conv'.format(idx)) else: raise ("activation function is not correct") return conv_rect diff --git a/video_prediction_savp/video_prediction/models/__init__.py b/video_prediction_savp/video_prediction/models/__init__.py index b71769a9d8cc523e4f108fc222fc2ed0284019f7..8010c4eeb2123fd94995c6474e7e1c8af6b02113 100644 --- a/video_prediction_savp/video_prediction/models/__init__.py +++ b/video_prediction_savp/video_prediction/models/__init__.py @@ -21,7 +21,6 @@ def get_model_class(model): 'vae': 'VanillaVAEVideoPredictionModel', 'convLSTM': 'VanillaConvLstmVideoPredictionModel', 'mcnet': 'McNetVideoPredictionModel', - 'convLSTM_Loliver': "ConvLstmLoliverVideoPredictionModel" } model_class = model_mappings.get(model, model) model_class = globals().get(model_class)