diff --git a/src/run_modules/model_setup.py b/src/run_modules/model_setup.py index 2512947b87b7042572d97ac132196b285977b1b8..4a72189283ff1bedc3014b29cdb752fe244c84bf 100644 --- a/src/run_modules/model_setup.py +++ b/src/run_modules/model_setup.py @@ -63,11 +63,11 @@ class ModelSetup(RunEnvironment): self.data_store.set("model", self.model, self.scope) def _set_checkpoint(self): + """ + Must be run after all callback functions that shall be tracked during training have been created (currently this + affects the learning rate decay and the advanced history [actually created in this method]). + """ lr = self.data_store.get("lr_decay", scope="general.model") - # checkpoint = ModelCheckpoint(filepath=self.checkpoint_name, verbose=1, monitor='val_loss', save_best_only=True, mode='auto') - # checkpoint = ModelCheckpointAdvanced(filepath=self.checkpoint_name, verbose=1, monitor='val_loss', - # save_best_only=True, mode='auto', callbacks_to_save=lr, - # callbacks_filepath=self.callbacks_name) hist = HistoryAdvanced() self.data_store.set("hist", hist, scope="general.model") callbacks = [{"callback": lr, "path": self.callbacks_name % "lr"}, diff --git a/src/run_modules/training.py b/src/run_modules/training.py index 7dfa06ffaacb4074637dd59dbcb913837e5fd00d..d1962605dc0a0eb3f6d0b0104f80e73a05134afc 100644 --- a/src/run_modules/training.py +++ b/src/run_modules/training.py @@ -89,7 +89,7 @@ class Training(RunEnvironment): # callbacks=self.callbacks) callbacks=[self.lr_sc, self.hist, self.checkpoint]) else: - lr_filepath = self.checkpoint.callbacks[0]["path"] # TODO: stopped here. why does training start 1 epoch too early or doesn't it? + lr_filepath = self.checkpoint.callbacks[0]["path"] hist_filepath = self.checkpoint.callbacks[1]["path"] lr_callbacks = pickle.load(open(lr_filepath, "rb")) hist_callbacks = pickle.load(open(hist_filepath, "rb")) @@ -101,7 +101,7 @@ class Training(RunEnvironment): {"callback": self.hist, "path": hist_filepath}] self.checkpoint.update_callbacks(callbacks) self.checkpoint.update_best(hist_callbacks) - self.hist = self.model.fit_generator(generator=self.train_set.distribute_on_batches(), + _ = self.model.fit_generator(generator=self.train_set.distribute_on_batches(), steps_per_epoch=len(self.train_set), epochs=self.epochs, verbose=2,