diff --git a/.gitignore b/.gitignore index 31cfe991f3cb38160cc5a2b330ca4725ebaa834f..11b7c159fdc450144f4103db66af1eabe42791a0 100644 --- a/.gitignore +++ b/.gitignore @@ -55,5 +55,6 @@ Thumbs.db htmlcov/ .pytest_cache /test/data/ +/test/test_modules/data/ report.html /TestExperiment/ diff --git a/run.py b/run.py index 572d8f9f9c25c08b60d2df035d8332c75375c975..ea8c04ebde02a80b899a356eb0f7794055abe2d6 100644 --- a/run.py +++ b/run.py @@ -8,6 +8,7 @@ import argparse from src.modules.experiment_setup import ExperimentSetup from src.modules.run_environment import RunEnvironment from src.modules.pre_processing import PreProcessing +from src.modules.model_setup import ModelSetup from src.modules.modules import Training, PostProcessing @@ -18,6 +19,8 @@ def main(parser_args): station_type='background') PreProcessing() + ModelSetup() + Training() PostProcessing() diff --git a/src/data_generator.py b/src/data_generator.py index 860791235f111a7ffb151f2b06424be76dc8eba7..4e7dda9363c226c5fa92d03f1dbae6470e48d496 100644 --- a/src/data_generator.py +++ b/src/data_generator.py @@ -20,7 +20,7 @@ class DataGenerator(keras.utils.Sequence): def __init__(self, data_path: str, network: str, stations: Union[str, List[str]], variables: List[str], interpolate_dim: str, target_dim: str, target_var: str, station_type: str = None, - interpolate_method: str = "linear", limit_nan_fill: int = 1, window_history: int = 7, + interpolate_method: str = "linear", limit_nan_fill: int = 1, window_history_size: int = 7, window_lead_time: int = 4, transform_method: str = "standardise", **kwargs): self.data_path = os.path.abspath(data_path) self.network = network @@ -32,7 +32,7 @@ class DataGenerator(keras.utils.Sequence): self.station_type = station_type self.interpolate_method = interpolate_method self.limit_nan_fill = limit_nan_fill - self.window_history = window_history + self.window_history_size = window_history_size self.window_lead_time = window_lead_time self.transform_method = transform_method self.kwargs = kwargs @@ -100,7 +100,7 @@ class DataGenerator(keras.utils.Sequence): **self.kwargs) data.interpolate(self.interpolate_dim, method=self.interpolate_method, limit=self.limit_nan_fill) data.transform("datetime", method=self.transform_method) - data.make_history_window(self.interpolate_dim, self.window_history) + data.make_history_window(self.interpolate_dim, self.window_history_size) data.make_labels(self.target_dim, self.target_var, self.interpolate_dim, self.window_lead_time) data.history_label_nan_remove(self.interpolate_dim) return data diff --git a/src/inception_model.py b/src/inception_model.py index 64b5e09674dd9fccd8282b80c4fb9e87bbd5ea5c..126dc15320f4b0ac7ff660b709601f083700d01f 100644 --- a/src/inception_model.py +++ b/src/inception_model.py @@ -3,6 +3,7 @@ __date__ = '2019-10-22' import keras import keras.layers as layers +import logging class InceptionModelBase: @@ -51,7 +52,7 @@ class InceptionModelBase: regularizer = kwargs.get('regularizer', keras.regularizers.l2(0.01)) bn_settings = kwargs.get('bn_settings', {}) act_settings = kwargs.get('act_settings', {}) - print(f'Inception Block with activation: {activation}') + logging.debug(f'Inception Block with activation: {activation}') block_name = f'Block_{self.number_of_blocks}{self.block_part_name()}_{tower_kernel[0]}x{tower_kernel[1]}' diff --git a/src/modules/experiment_setup.py b/src/modules/experiment_setup.py index a20f0b83e9828550d2f717502b5371c2c1ad7e9a..472173c7df371bcb139672d02bfa6723dbd0658e 100644 --- a/src/modules/experiment_setup.py +++ b/src/modules/experiment_setup.py @@ -28,7 +28,7 @@ class ExperimentSetup(RunEnvironment): """ def __init__(self, parser_args=None, var_all_dict=None, stations=None, network=None, station_type=None, variables=None, - statistics_per_var=None, start=None, end=None, window_history=None, target_var="o3", target_dim=None, + statistics_per_var=None, start=None, end=None, window_history_size=None, target_var="o3", target_dim=None, window_lead_time=None, dimensions=None, interpolate_dim=None, interpolate_method=None, limit_nan_fill=None, train_start=None, train_end=None, val_start=None, val_end=None, test_start=None, test_end=None, use_all_stations_on_all_data_sets=True, trainable=False, fraction_of_train=None, @@ -58,7 +58,7 @@ class ExperimentSetup(RunEnvironment): self._set_param("statistics_per_var", statistics_per_var, default=self.data_store.get("var_all_dict", "general")) self._set_param("start", start, default="1997-01-01", scope="general") self._set_param("end", end, default="2017-12-31", scope="general") - self._set_param("window_history", window_history, default=13) + self._set_param("window_history_size", window_history_size, default=13) # target self._set_param("target_var", target_var, default="o3") diff --git a/src/modules/pre_processing.py b/src/modules/pre_processing.py index 2b830f97bcb67251864003357a87762bcdeca07b..764613ea4558bfdef4cbada668f16608f81d5f95 100644 --- a/src/modules/pre_processing.py +++ b/src/modules/pre_processing.py @@ -13,7 +13,7 @@ from src.join import EmptyQueryResult DEFAULT_ARGS_LIST = ["data_path", "network", "stations", "variables", "interpolate_dim", "target_dim", "target_var"] -DEFAULT_KWARGS_LIST = ["limit_nan_fill", "window_history", "window_lead_time", "statistics_per_var", "station_type"] +DEFAULT_KWARGS_LIST = ["limit_nan_fill", "window_history_size", "window_lead_time", "statistics_per_var", "station_type"] class PreProcessing(RunEnvironment): diff --git a/test/test_data_generator.py b/test/test_data_generator.py index 2fe8b8c0b5a7f4f8be9626b0061702acb53ecb6b..6801e064be804da0368520ab0ef81bdba8bca2d3 100644 --- a/test/test_data_generator.py +++ b/test/test_data_generator.py @@ -21,7 +21,7 @@ class TestDataGenerator: assert gen.target_var == 'o3' assert gen.interpolate_method == "linear" assert gen.limit_nan_fill == 1 - assert gen.window_history == 7 + assert gen.window_history_size == 7 assert gen.window_lead_time == 4 assert gen.transform_method == "standardise" assert gen.kwargs == {} @@ -44,7 +44,7 @@ class TestDataGenerator: assert station[0].Stations.data == "DEBW107" assert station[0].data.shape[1:] == (8, 1, 2) assert station[1].data.shape[-1] == gen.window_lead_time - assert station[0].data.shape[1] == gen.window_history + 1 + assert station[0].data.shape[1] == gen.window_history_size + 1 def test_iter(self, gen): assert hasattr(gen, '_iterator') is False diff --git a/test/test_modules/test_experiment_setup.py b/test/test_modules/test_experiment_setup.py index be3db59e2415f28ea63d42b7cc6ced6b2c095700..e1ec57f9e6cace563b6311ee0c1f34fefcb2c7c2 100644 --- a/test/test_modules/test_experiment_setup.py +++ b/test/test_modules/test_experiment_setup.py @@ -72,7 +72,7 @@ class TestExperimentSetup: assert data_store.get("statistics_per_var", "general") == default_var_all_dict assert data_store.get("start", "general") == "1997-01-01" assert data_store.get("end", "general") == "2017-12-31" - assert data_store.get("window_history", "general") == 13 + assert data_store.get("window_history_size", "general") == 13 # target assert data_store.get("target_var", "general") == "o3" assert data_store.get("target_dim", "general") == "variables" @@ -100,7 +100,7 @@ class TestExperimentSetup: var_all_dict={'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum'}, stations=['DEBY053', 'DEBW059', 'DEBW027'], network="INTERNET", station_type="background", variables=["o3", "temp"], - statistics_per_var=None, start="1999-01-01", end="2001-01-01", window_history=4, + statistics_per_var=None, start="1999-01-01", end="2001-01-01", window_history_size=4, target_var="temp", target_dim="target", window_lead_time=10, dimensions="dim1", interpolate_dim="int_dim", interpolate_method="cubic", limit_nan_fill=5, train_start="2000-01-01", train_end="2000-01-02", val_start="2000-01-03", val_end="2000-01-04", test_start="2000-01-05", @@ -127,7 +127,7 @@ class TestExperimentSetup: 'temp': 'maximum'} assert data_store.get("start", "general") == "1999-01-01" assert data_store.get("end", "general") == "2001-01-01" - assert data_store.get("window_history", "general") == 4 + assert data_store.get("window_history_size", "general") == 4 # target assert data_store.get("target_var", "general") == "temp" assert data_store.get("target_dim", "general") == "target"