Merge branch 'bing_issue#078_allow_empty_checkpoint' into develop

video_prediction_tools/HPC_scripts/train_model_era5_template.sh

@@ -50,8 +50,5 @@ dataset=era5
 
 # run training
 srun python ../main_scripts/main_train_models.py --input_dir  ${source_dir} --datasplit_dict ${datasplit_dict} \
- --dataset ${dataset}  --model ${model} --model_hparams_dict ${model_hparams} --output_dir ${destination_dir}
-
-
-
+ --dataset ${dataset}  --model ${model} --model_hparams_dict ${model_hparams} --output_dir ${destination_dir} --checkpoint_dir  ${destination_dir}
  

video_prediction_tools/main_scripts/main_train_models.py

@@ -259,7 +259,7 @@ class TrainModel(object):
         """
         Restore the train and validation losses in the pickle file if checkpoint is given 
         """
-        if self.start_step == 0:
+        if not os.path.exists(os.path.join(self.output_dir,"checkpoint")):
             train_losses = []
             val_losses = []
         else:
@@ -278,6 +278,7 @@ class TrainModel(object):
             print("parameter_count =", sess.run(self.parameter_count))
             sess.run(tf.global_variables_initializer())
             sess.run(tf.local_variables_initializer())
+            if os.path.exists(os.path.join(self.output_dir,"checkpoint")):
                 self.restore(sess, self.checkpoint)
             #sess.graph.finalize()
             self.start_step = sess.run(self.global_step)