Skip to content
Snippets Groups Projects
Commit e3b98733 authored by lukas leufen's avatar lukas leufen
Browse files

new name for tmp pickle data, to distinguish different subsets

parent ebc4ff73
Branches
Tags
2 merge requests!37include new development,!35Lukas issue046 feat time series plot
...@@ -90,8 +90,7 @@ class DataGenerator(keras.utils.Sequence): ...@@ -90,8 +90,7 @@ class DataGenerator(keras.utils.Sequence):
:return: The generator's time series of history data and its labels :return: The generator's time series of history data and its labels
""" """
data = self.get_data_generator(key=item) data = self.get_data_generator(key=item)
return data.history.transpose("datetime", "window", "Stations", "variables"), \ return data.get_transposed_history(), data.label.squeeze("Stations").transpose("datetime", "window")
data.label.squeeze("Stations").transpose("datetime", "window")
def get_data_generator(self, key: Union[str, int] = None, local_tmp_storage: bool = True) -> DataPrep: def get_data_generator(self, key: Union[str, int] = None, local_tmp_storage: bool = True) -> DataPrep:
""" """
...@@ -124,7 +123,10 @@ class DataGenerator(keras.utils.Sequence): ...@@ -124,7 +123,10 @@ class DataGenerator(keras.utils.Sequence):
Save given data locally as .pickle in self.data_path_tmp with name '<station>_<var1>_<var2>_..._<varX>.pickle' Save given data locally as .pickle in self.data_path_tmp with name '<station>_<var1>_<var2>_..._<varX>.pickle'
:param data: any data, that should be saved :param data: any data, that should be saved
""" """
file = os.path.join(self.data_path_tmp, f"{''.join(data.station)}_{'_'.join(sorted(data.variables))}.pickle") date = f"{self.kwargs.get('start')}_{self.kwargs.get('end')}"
vars = '_'.join(sorted(data.variables))
station = ''.join(data.station)
file = os.path.join(self.data_path_tmp, f"{station}_{vars}_{date}_.pickle")
with open(file, "wb") as f: with open(file, "wb") as f:
pickle.dump(data, f) pickle.dump(data, f)
logging.debug(f"save pickle data to {file}") logging.debug(f"save pickle data to {file}")
...@@ -136,7 +138,10 @@ class DataGenerator(keras.utils.Sequence): ...@@ -136,7 +138,10 @@ class DataGenerator(keras.utils.Sequence):
:param variables: list of variables to load :param variables: list of variables to load
:return: loaded data :return: loaded data
""" """
file = os.path.join(self.data_path_tmp, f"{''.join(station)}_{'_'.join(sorted(variables))}.pickle") date = f"{self.kwargs.get('start')}_{self.kwargs.get('end')}"
vars = '_'.join(sorted(variables))
station = ''.join(station)
file = os.path.join(self.data_path_tmp, f"{station}_{vars}_{date}_.pickle")
with open(file, "rb") as f: with open(file, "rb") as f:
data = pickle.load(f) data = pickle.load(f)
logging.debug(f"load pickle data from {file}") logging.debug(f"load pickle data from {file}")
......
...@@ -385,6 +385,10 @@ class DataPrep(object): ...@@ -385,6 +385,10 @@ class DataPrep(object):
data.loc[..., used_chem_vars] = data.loc[..., used_chem_vars].clip(min=minimum) data.loc[..., used_chem_vars] = data.loc[..., used_chem_vars].clip(min=minimum)
return data return data
def get_transposed_history(self):
if self.history is not None:
return self.history.transpose("datetime", "window", "Stations", "variables")
if __name__ == "__main__": if __name__ == "__main__":
dp = DataPrep('data/', 'dummy', 'DEBW107', ['o3', 'temp'], statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'}) dp = DataPrep('data/', 'dummy', 'DEBW107', ['o3', 'temp'], statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'})
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment