From e3b987337991a496f86b9b037b65dcebddb94c9d Mon Sep 17 00:00:00 2001
From: lukas leufen <l.leufen@fz-juelich.de>
Date: Mon, 10 Feb 2020 16:24:34 +0100
Subject: [PATCH] new name for tmp pickle data, to distinguish different
subsets
---
src/data_handling/data_generator.py | 13 +++++++++----
src/data_handling/data_preparation.py | 4 ++++
2 files changed, 13 insertions(+), 4 deletions(-)
diff --git a/src/data_handling/data_generator.py b/src/data_handling/data_generator.py
index 732a7efd..92ff8b71 100644
--- a/src/data_handling/data_generator.py
+++ b/src/data_handling/data_generator.py
@@ -90,8 +90,7 @@ class DataGenerator(keras.utils.Sequence):
:return: The generator's time series of history data and its labels
"""
data = self.get_data_generator(key=item)
- return data.history.transpose("datetime", "window", "Stations", "variables"), \
- data.label.squeeze("Stations").transpose("datetime", "window")
+ return data.get_transposed_history(), data.label.squeeze("Stations").transpose("datetime", "window")
def get_data_generator(self, key: Union[str, int] = None, local_tmp_storage: bool = True) -> DataPrep:
"""
@@ -124,7 +123,10 @@ class DataGenerator(keras.utils.Sequence):
Save given data locally as .pickle in self.data_path_tmp with name '<station>_<var1>_<var2>_..._<varX>.pickle'
:param data: any data, that should be saved
"""
- file = os.path.join(self.data_path_tmp, f"{''.join(data.station)}_{'_'.join(sorted(data.variables))}.pickle")
+ date = f"{self.kwargs.get('start')}_{self.kwargs.get('end')}"
+ vars = '_'.join(sorted(data.variables))
+ station = ''.join(data.station)
+ file = os.path.join(self.data_path_tmp, f"{station}_{vars}_{date}_.pickle")
with open(file, "wb") as f:
pickle.dump(data, f)
logging.debug(f"save pickle data to {file}")
@@ -136,7 +138,10 @@ class DataGenerator(keras.utils.Sequence):
:param variables: list of variables to load
:return: loaded data
"""
- file = os.path.join(self.data_path_tmp, f"{''.join(station)}_{'_'.join(sorted(variables))}.pickle")
+ date = f"{self.kwargs.get('start')}_{self.kwargs.get('end')}"
+ vars = '_'.join(sorted(variables))
+ station = ''.join(station)
+ file = os.path.join(self.data_path_tmp, f"{station}_{vars}_{date}_.pickle")
with open(file, "rb") as f:
data = pickle.load(f)
logging.debug(f"load pickle data from {file}")
diff --git a/src/data_handling/data_preparation.py b/src/data_handling/data_preparation.py
index c39625b1..81ce5cdd 100644
--- a/src/data_handling/data_preparation.py
+++ b/src/data_handling/data_preparation.py
@@ -385,6 +385,10 @@ class DataPrep(object):
data.loc[..., used_chem_vars] = data.loc[..., used_chem_vars].clip(min=minimum)
return data
+ def get_transposed_history(self):
+ if self.history is not None:
+ return self.history.transpose("datetime", "window", "Stations", "variables")
+
if __name__ == "__main__":
dp = DataPrep('data/', 'dummy', 'DEBW107', ['o3', 'temp'], statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'})
--
GitLab