Skip to content
Snippets Groups Projects
Commit 2059a593 authored by leufen1's avatar leufen1
Browse files

more skip args for snapshot check

parent 4457e8e0
Branches
Tags
4 merge requests!468first implementation of toar-data-v2, can load data (but cannot process these...,!467Resolve "release v2.2.0",!461Merge Dev into issue400,!454Resolve "REFAC: improve snapshot"
Pipeline #106004 passed
......@@ -170,8 +170,6 @@ def check_nested_equality(obj1, obj2, precision=None, skip_args=None):
message = f"{obj1}!={obj2}\n{obj1} and {obj2} do not match"
assert obj1 == obj2
except AssertionError:
message = message.split("\n")
logging.info(message[0])
logging.debug(message[1])
logging.info(message)
return False
return True
......@@ -412,19 +412,33 @@ class PreProcessing(RunEnvironment):
logging.info(f"load snapshot for preprocessing from {file}")
with open(file, "rb") as f:
snapshot = dill.load(f)
excluded_params = ["batch_path", "batch_size", "block_length", "bootstrap_method", "bootstrap_path",
"bootstrap_type", "competitor_path", "competitors", "create_new_bootstraps",
"create_new_model", "create_snapshot", "data_collection", "debug_mode",
"do_uncertainty_estimate", "early_stopping_epochs", "epochs", "evaluate_competitors",
"evaluate_feature_importance", "experiment_name", "experiment_path", "forecast_path",
"fraction_of_training", "hostname", "hpc_hosts", "log_level_stream", "logging_path",
"login_nodes", "max_number_multiprocessing", "model_class", "model_path", "n_boots",
"neighbors", "plot_list", "plot_path", "restore_best_model_weights", "snapshot_load_path",
"snapshot_path", "stations", "tmp_path", "train_model", "transformation",
"use_multiprocessing", ]
excluded_params = ["activation", "activation_output", "add_dense_layer", "batch_normalization", "batch_path",
"batch_size", "block_length", "bootstrap_method", "bootstrap_path", "bootstrap_type",
"competitor_path", "competitors", "create_new_bootstraps", "create_new_model",
"create_snapshot", "data_collection", "debug_mode", "dense_layer_configuration",
"do_uncertainty_estimate", "dropout", "dropout_rnn", "early_stopping_epochs", "epochs",
"evaluate_competitors", "evaluate_feature_importance", "experiment_name", "experiment_path",
"exponent_last_layer", "forecast_path", "fraction_of_training", "hostname", "hpc_hosts",
"kernel_regularizer", "kernel_size", "layer_configuration", "log_level_stream",
"logging_path", "login_nodes", "loss_type", "loss_weights", "max_number_multiprocessing",
"model_class", "model_display_name", "model_path", "n_boots", "n_hidden", "n_layer",
"neighbors", "plot_list", "plot_path", "regularizer", "restore_best_model_weights",
"snapshot_load_path", "snapshot_path", "stations", "tmp_path", "train_model",
"transformation", "use_multiprocessing", ]
# excluded_params = ["batch_path", "batch_size", "block_length", "bootstrap_method", "bootstrap_path",
# "bootstrap_type", "competitor_path", "competitors", "create_new_bootstraps",
# "create_new_model", "create_snapshot", "data_collection", "debug_mode",
# "do_uncertainty_estimate", "early_stopping_epochs", "epochs", "evaluate_competitors",
# "evaluate_feature_importance", "experiment_name", "experiment_path", "forecast_path",
# "fraction_of_training", "hostname", "hpc_hosts", "log_level_stream", "logging_path",
# "login_nodes", "max_number_multiprocessing", "model_class", "model_path", "n_boots",
# "neighbors", "plot_list", "plot_path", "restore_best_model_weights", "snapshot_load_path",
# "snapshot_path", "stations", "tmp_path", "train_model", "transformation",
# "use_multiprocessing", ]
data_handler = self.data_store.get("data_handler")
excluded_params = list(set(excluded_params + data_handler.store_attributes()))
model_class = self.data_store.get("model_class")
excluded_params = list(set(excluded_params + data_handler.store_attributes() + model_class.requirements()))
if check_nested_equality(self.data_store._store, snapshot._store, skip_args=excluded_params) is True:
self.update_datastore(snapshot, excluded_params=remove_items(excluded_params, ["transformation",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment