Skip to content
Snippets Groups Projects

Resolve "store and load local clim apriori data"

1 file
+ 25
0
Compare changes
  • Side-by-side
  • Inline
@@ -64,6 +64,7 @@ class PreProcessing(RunEnvironment):
if snapshot_load_path is None:
stations = self.data_store.get("stations")
data_handler = self.data_store.get("data_handler")
self._load_apriori()
_, valid_stations = self.validate_station(data_handler, stations,
"preprocessing") # , store_processed_data=False)
if len(valid_stations) == 0:
@@ -318,6 +319,30 @@ class PreProcessing(RunEnvironment):
attrs[k] = dict(attrs.get(k, {}), **{station: v})
for k, v in attrs.items():
self.data_store.set(k, v)
self._store_apriori()
def _store_apriori(self):
apriori = self.data_store.get_default("apriori", default=None)
if apriori:
experiment_path = self.data_store.get("experiment_path")
path = os.path.join(experiment_path, "data", "apriori")
store_file = os.path.join(path, "apriori.pickle")
if not os.path.exists(path):
path_config.check_path_and_create(path)
with open(store_file, "wb") as f:
dill.dump(apriori, f, protocol=4)
logging.debug(f"Store apriori options locally for later use at: {store_file}")
def _load_apriori(self):
if self.data_store.get_default("apriori", default=None) is None:
apriori_file = self.data_store.get_default("apriori_file", None)
if apriori_file is not None:
if os.path.exists(apriori_file):
logging.info(f"use apriori data from given file: {apriori_file}")
with open(apriori_file, "rb") as pickle_file:
self.data_store.set("apriori", dill.load(pickle_file))
else:
logging.info(f"cannot load apriori file: {apriori_file}. Use fresh calculation from data.")
def transformation(self, data_handler: AbstractDataHandler, stations):
calculate_fresh_transformation = self.data_store.get_default("calculate_fresh_transformation", True)
Loading