diff --git a/run.py b/run.py index 9f38fdca9c51cbed332725ce8e120e1493551b93..8e4d9c46f4a39224335dd65b689f519943166d0f 100644 --- a/run.py +++ b/run.py @@ -2,15 +2,15 @@ __author__ = "Lukas Leufen" __date__ = '2019-11-14' -import logging import argparse +import logging from src.run_modules.experiment_setup import ExperimentSetup -from src.run_modules.run_environment import RunEnvironment -from src.run_modules.pre_processing import PreProcessing from src.run_modules.model_setup import ModelSetup -from src.run_modules.training import Training from src.run_modules.post_processing import PostProcessing +from src.run_modules.pre_processing import PreProcessing +from src.run_modules.run_environment import RunEnvironment +from src.run_modules.training import Training def main(parser_args): diff --git a/src/data_handling/data_generator.py b/src/data_handling/data_generator.py index 1de0ab2092b46dfca6281963b260b1fc6bc65387..0e60c8916c62c10929fad72b2271b0e7f54dee4a 100644 --- a/src/data_handling/data_generator.py +++ b/src/data_handling/data_generator.py @@ -1,13 +1,15 @@ __author__ = 'Felix Kleinert, Lukas Leufen' __date__ = '2019-11-07' -import keras -from src import helpers -from src.data_handling.data_preparation import DataPrep import os from typing import Union, List, Tuple + +import keras import xarray as xr +from src import helpers +from src.data_handling.data_preparation import DataPrep + class DataGenerator(keras.utils.Sequence): """ diff --git a/src/data_handling/data_preparation.py b/src/data_handling/data_preparation.py index e3506724eb5f78a6cd7af6217c768b183d671dba..1ae5f0ae8ec77c9e5e0b6776d4e17f1dff412286 100644 --- a/src/data_handling/data_preparation.py +++ b/src/data_handling/data_preparation.py @@ -1,15 +1,17 @@ __author__ = 'Felix Kleinert, Lukas Leufen' __date__ = '2019-10-16' -import xarray as xr -import pandas as pd -import numpy as np +import datetime as dt import logging import os +from typing import Union, List, Iterable + +import numpy as np +import pandas as pd +import xarray as xr + from src import join, helpers from src import statistics -from typing import Union, List, Iterable -import datetime as dt # define a more general date type for type hinting date = Union[dt.date, dt.datetime] diff --git a/src/datastore.py b/src/datastore.py index 69e486796cc4320d88b3c3f5fd4a970d4ee814e9..d9f844ff97acb3f5c6600205f91100219d9c53e6 100644 --- a/src/datastore.py +++ b/src/datastore.py @@ -2,8 +2,8 @@ __author__ = 'Lukas Leufen' __date__ = '2019-11-22' -from typing import Any, List, Tuple, Dict from abc import ABC +from typing import Any, List, Tuple, Dict class NameNotFoundInDataStore(Exception): diff --git a/src/join.py b/src/join.py index bcbce84d8a20356998e547b025f95f9c80e54b13..351060f7bf4949801f94b04c13e3881f008389b6 100644 --- a/src/join.py +++ b/src/join.py @@ -2,11 +2,13 @@ __author__ = 'Felix Kleinert, Lukas Leufen' __date__ = '2019-10-16' -import requests -import logging -import pandas as pd import datetime as dt +import logging from typing import Iterator, Union, List, Dict + +import pandas as pd +import requests + from src import helpers from src.join_settings import join_settings diff --git a/src/model_modules/flatten.py b/src/model_modules/flatten.py index 707b0d4421fd506d16f93d7c56406abc878ab9f1..bbe92472ebb48e7486dede099dc098a161f51695 100644 --- a/src/model_modules/flatten.py +++ b/src/model_modules/flatten.py @@ -1,9 +1,10 @@ __author__ = "Felix Kleinert, Lukas Leufen" __date__ = '2019-12-02' -import keras from typing import Callable +import keras + def flatten_tail(input_X: keras.layers, name: str, bound_weight: bool = False, dropout_rate: float = 0.0, window_lead_time: int = 4, activation: Callable = keras.activations.relu, diff --git a/src/model_modules/inception_model.py b/src/model_modules/inception_model.py index 126dc15320f4b0ac7ff660b709601f083700d01f..11093df56a4b262f75eae7a6f7c05e6e44e1435d 100644 --- a/src/model_modules/inception_model.py +++ b/src/model_modules/inception_model.py @@ -1,9 +1,10 @@ __author__ = 'Felix Kleinert, Lukas Leufen' __date__ = '2019-10-22' +import logging + import keras import keras.layers as layers -import logging class InceptionModelBase: diff --git a/src/model_modules/linear_model.py b/src/model_modules/linear_model.py index d8c455b551ef748e90ae601c91931a8092a82b38..3d5323e1b0303b497c1f26c4e84ee9b968380425 100644 --- a/src/model_modules/linear_model.py +++ b/src/model_modules/linear_model.py @@ -2,8 +2,8 @@ __author__ = "Felix Kleinert, Lukas Leufen" __date__ = '2019-12-11' -import statsmodels.api as sm import numpy as np +import statsmodels.api as sm class OrdinaryLeastSquaredModel: diff --git a/src/model_modules/model_class.py b/src/model_modules/model_class.py index 02f43dc1b208cfd8a52a937298217216f26fbdb6..bb4801acfb5c3a643aecbcfad9cfdb758258d0ef 100644 --- a/src/model_modules/model_class.py +++ b/src/model_modules/model_class.py @@ -9,8 +9,6 @@ from typing import Any, Callable import keras -from src import helpers - class AbstractModelClass(ABC): diff --git a/src/plotting/postprocessing_plotting.py b/src/plotting/postprocessing_plotting.py index ece1ad97fb0be754c97997fb63d711e351d1095f..6ae8573d11375335edda48a1bf26f30d6125bd69 100644 --- a/src/plotting/postprocessing_plotting.py +++ b/src/plotting/postprocessing_plotting.py @@ -1,26 +1,25 @@ __author__ = "Lukas Leufen, Felix Kleinert" __date__ = '2019-12-17' -import os import logging import math +import os import warnings -from src import helpers -from src.helpers import TimeTracking -from src.run_modules.run_environment import RunEnvironment +from typing import Dict, List, Tuple +import cartopy.crs as ccrs +import cartopy.feature as cfeature +import matplotlib +import matplotlib.pyplot as plt import numpy as np -import xarray as xr import pandas as pd - -import matplotlib import seaborn as sns -import matplotlib.pyplot as plt -import cartopy.crs as ccrs -import cartopy.feature as cfeature +import xarray as xr from matplotlib.backends.backend_pdf import PdfPages -from typing import Dict, List, Tuple +from src import helpers +from src.helpers import TimeTracking +from src.run_modules.run_environment import RunEnvironment logging.getLogger('matplotlib').setLevel(logging.WARNING) diff --git a/src/plotting/training_monitoring.py b/src/plotting/training_monitoring.py index 339ba63711cde8122f2879d66b66d490429e6d23..7e656895c5eecdabe1ef26869b68fb9494ed4c8c 100644 --- a/src/plotting/training_monitoring.py +++ b/src/plotting/training_monitoring.py @@ -5,9 +5,9 @@ __date__ = '2019-12-11' from typing import Union, Dict, List import keras -import pandas as pd import matplotlib import matplotlib.pyplot as plt +import pandas as pd from src.model_modules.keras_extensions import LearningRateDecay diff --git a/src/run_modules/experiment_setup.py b/src/run_modules/experiment_setup.py index d2410de06c8726ad1aef3626a56876c3202bf334..c995367f06014ab3393317c48cb6ec2229ee21df 100644 --- a/src/run_modules/experiment_setup.py +++ b/src/run_modules/experiment_setup.py @@ -2,15 +2,14 @@ __author__ = "Lukas Leufen, Felix Kleinert" __date__ = '2019-11-15' -import logging import argparse -from typing import Union, Dict, Any +import logging import os +from typing import Union, Dict, Any from src import helpers from src.run_modules.run_environment import RunEnvironment - DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBY052', 'DEBY032', 'DEBW022', 'DEBY004', 'DEBY020', 'DEBW030', 'DEBW037', 'DEBW031', 'DEBW015', 'DEBW073', 'DEBY039', 'DEBW038', 'DEBW081', 'DEBY075', 'DEBW040', 'DEBY053', 'DEBW059', 'DEBW027', 'DEBY072', 'DEBW042', 'DEBW039', 'DEBY001', diff --git a/src/run_modules/model_setup.py b/src/run_modules/model_setup.py index 4a72189283ff1bedc3014b29cdb752fe244c84bf..7049af591cdf73434459d4c3f5f6c11e80ab64c0 100644 --- a/src/run_modules/model_setup.py +++ b/src/run_modules/model_setup.py @@ -2,19 +2,20 @@ __author__ = "Lukas Leufen, Felix Kleinert" __date__ = '2019-12-02' -import keras -from keras import losses -import tensorflow as tf import logging import os -from src.run_modules.run_environment import RunEnvironment +import keras +import tensorflow as tf +from keras import losses + from src.helpers import l_p_loss -from src.model_modules.keras_extensions import HistoryAdvanced, ModelCheckpointAdvanced -from src.model_modules.inception_model import InceptionModelBase from src.model_modules.flatten import flatten_tail +from src.model_modules.inception_model import InceptionModelBase +from src.model_modules.keras_extensions import HistoryAdvanced, ModelCheckpointAdvanced # from src.model_modules.model_class import MyBranchedModel as MyModel from src.model_modules.model_class import MyLittleModel as MyModel +from src.run_modules.run_environment import RunEnvironment class ModelSetup(RunEnvironment): diff --git a/src/run_modules/post_processing.py b/src/run_modules/post_processing.py index 93cb27dcdc7ad10137dd663c35ca43c401254872..258e0e0c9a53652fedee36ad16d24019c6a4775d 100644 --- a/src/run_modules/post_processing.py +++ b/src/run_modules/post_processing.py @@ -5,20 +5,21 @@ __date__ = '2019-12-11' import logging import os +import keras import numpy as np import pandas as pd import xarray as xr -import keras -from src.run_modules.run_environment import RunEnvironment +from src import statistics from src.data_handling.data_distributor import Distributor from src.data_handling.data_generator import DataGenerator -from src.model_modules.linear_model import OrdinaryLeastSquaredModel -from src import statistics -from src.plotting.postprocessing_plotting import plot_conditional_quantiles -from src.plotting.postprocessing_plotting import PlotMonthlySummary, PlotStationMap, PlotClimatologicalSkillScore, PlotCompetitiveSkillScore from src.datastore import NameNotFoundInDataStore from src.helpers import TimeTracking +from src.model_modules.linear_model import OrdinaryLeastSquaredModel +from src.plotting.postprocessing_plotting import PlotMonthlySummary, PlotStationMap, PlotClimatologicalSkillScore, \ + PlotCompetitiveSkillScore +from src.plotting.postprocessing_plotting import plot_conditional_quantiles +from src.run_modules.run_environment import RunEnvironment class PostProcessing(RunEnvironment): diff --git a/src/run_modules/pre_processing.py b/src/run_modules/pre_processing.py index c5b1c53fc007a887f5219066185b5a02f352ee7c..9eef12f5dafbf2930ed22337c46281954de55b3c 100644 --- a/src/run_modules/pre_processing.py +++ b/src/run_modules/pre_processing.py @@ -7,9 +7,8 @@ from typing import Tuple, Dict, List from src.data_handling.data_generator import DataGenerator from src.helpers import TimeTracking -from src.run_modules.run_environment import RunEnvironment from src.join import EmptyQueryResult - +from src.run_modules.run_environment import RunEnvironment DEFAULT_ARGS_LIST = ["data_path", "network", "stations", "variables", "interpolate_dim", "target_dim", "target_var"] DEFAULT_KWARGS_LIST = ["limit_nan_fill", "window_history_size", "window_lead_time", "statistics_per_var", diff --git a/src/run_modules/training.py b/src/run_modules/training.py index 99afd8300ca28fec3a589e69fa5b4eff1a37914a..e2a98f27c65e6050b0edae2bbc178abbf97ab646 100644 --- a/src/run_modules/training.py +++ b/src/run_modules/training.py @@ -1,16 +1,17 @@ __author__ = "Lukas Leufen, Felix Kleinert" __date__ = '2019-12-05' +import json import logging import os -import json -import keras import pickle -from src.run_modules.run_environment import RunEnvironment +import keras + from src.data_handling.data_distributor import Distributor -from src.plotting.training_monitoring import PlotModelHistory, PlotModelLearningRate from src.model_modules.keras_extensions import LearningRateDecay, ModelCheckpointAdvanced +from src.plotting.training_monitoring import PlotModelHistory, PlotModelLearningRate +from src.run_modules.run_environment import RunEnvironment class Training(RunEnvironment): diff --git a/test/test_data_handling/test_data_distributor.py b/test/test_data_handling/test_data_distributor.py index cb51f20c8771ec49116731f02c7b462a62405394..4c6dbb1c38f2e4a49e53883fbe3cb33cb565118a 100644 --- a/test/test_data_handling/test_data_distributor.py +++ b/test/test_data_handling/test_data_distributor.py @@ -1,6 +1,5 @@ import math import os -import shutil import keras import numpy as np diff --git a/test/test_data_handling/test_data_generator.py b/test/test_data_handling/test_data_generator.py index 879436afddb8da8d11d6cc585da7c703aa12ef8a..2f741a1aca241767681f23f2e0912a28103b24b7 100644 --- a/test/test_data_handling/test_data_generator.py +++ b/test/test_data_handling/test_data_generator.py @@ -1,6 +1,7 @@ -import pytest import os -import shutil + +import pytest + from src.data_handling.data_generator import DataGenerator diff --git a/test/test_data_handling/test_data_preparation.py b/test/test_data_handling/test_data_preparation.py index 12b619d9e31990f6cc24216ff84ad9d030265e36..8b5968460a935638ba4d64d3e939eb32d6b49189 100644 --- a/test/test_data_handling/test_data_preparation.py +++ b/test/test_data_handling/test_data_preparation.py @@ -1,12 +1,14 @@ -import pytest +import datetime as dt import os -from src.data_handling.data_preparation import DataPrep -from src.join import EmptyQueryResult +from operator import itemgetter + import numpy as np -import xarray as xr -import datetime as dt import pandas as pd -from operator import itemgetter +import pytest +import xarray as xr + +from src.data_handling.data_preparation import DataPrep +from src.join import EmptyQueryResult class TestDataPrep: diff --git a/test/test_datastore.py b/test/test_datastore.py index e7510cffacafb4a6db6006097f48992fb6a10e55..95a58deafc915dd6193960e77bb99cc8ab8d85cb 100644 --- a/test/test_datastore.py +++ b/test/test_datastore.py @@ -2,9 +2,10 @@ __author__ = 'Lukas Leufen' __date__ = '2019-11-22' +import pytest + from src.datastore import AbstractDataStore, DataStoreByVariable, DataStoreByScope from src.datastore import NameNotFoundInDataStore, NameNotFoundInScope, EmptyScope -import pytest class TestAbstractDataStore: diff --git a/test/test_helpers.py b/test/test_helpers.py index 463007dc361d21df934bb239b3ecac2fc86882ad..b807d2b8612b9ee006bff43f1ae4cfcfd2dd07e1 100644 --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -1,13 +1,13 @@ -import pytest -from src.helpers import * import logging import os +import platform + import keras -import numpy as np import mock -import platform +import numpy as np +import pytest -from src.model_modules.keras_extensions import LearningRateDecay +from src.helpers import * class TestToList: diff --git a/test/test_join.py b/test/test_join.py index 029d7ec87efba1003b62b5c92361f574711355de..fe3d33d6296c16bfc72675bc1737aad12ee3c8b9 100644 --- a/test/test_join.py +++ b/test/test_join.py @@ -1,5 +1,5 @@ from typing import Iterable -import datetime as dt + import pytest from src.join import * diff --git a/test/test_model_modules/test_inception_model.py b/test/test_model_modules/test_inception_model.py index b2a923c47c4351ed4c2f543a4e30d25fdbaa58ea..aa5cb284ab196d733e04a9882fa4d5a4ef639a6d 100644 --- a/test/test_model_modules/test_inception_model.py +++ b/test/test_model_modules/test_inception_model.py @@ -1,6 +1,7 @@ +import keras import pytest + from src.model_modules.inception_model import InceptionModelBase -import keras class TestInceptionModelBase: diff --git a/test/test_model_modules/test_keras_extensions.py b/test/test_model_modules/test_keras_extensions.py index c50e5e425779575eb2b492213a0b39b2b7c3376e..8bcebec60efe46db48a403b72e61a2036a6c00b9 100644 --- a/test/test_model_modules/test_keras_extensions.py +++ b/test/test_model_modules/test_keras_extensions.py @@ -1,8 +1,9 @@ -import pytest -from src.model_modules.keras_extensions import * -from src.helpers import l_p_loss import keras import numpy as np +import pytest + +from src.helpers import l_p_loss +from src.model_modules.keras_extensions import * class TestLearningRateDecay: diff --git a/test/test_model_modules/test_model_class.py b/test/test_model_modules/test_model_class.py index 0af16012336c9dbcc3133d7dac1f365e276d11bc..0dbd2d9b67a0748bf09eb4f59e1888aae1ea405d 100644 --- a/test/test_model_modules/test_model_class.py +++ b/test/test_model_modules/test_model_class.py @@ -1,5 +1,5 @@ -import pytest import keras +import pytest from src.model_modules.model_class import AbstractModelClass diff --git a/test/test_modules/test_experiment_setup.py b/test/test_modules/test_experiment_setup.py index 08062f4c81dc172d47daa0c0592f07f4da254fb0..7a4f16fd055e0af0aea95181d475d061c185ca92 100644 --- a/test/test_modules/test_experiment_setup.py +++ b/test/test_modules/test_experiment_setup.py @@ -1,11 +1,11 @@ -import pytest -import logging import argparse +import logging import os -from src.run_modules.experiment_setup import ExperimentSetup +import pytest + from src.helpers import TimeTracking, prepare_host -from src.datastore import NameNotFoundInScope, NameNotFoundInDataStore +from src.run_modules.experiment_setup import ExperimentSetup class TestExperimentSetup: diff --git a/test/test_modules/test_model_setup.py b/test/test_modules/test_model_setup.py index 65c683003d173031115164b4800d7759ff9cec2f..2864ae45bcd7d3c6109d6d84fe5ea152a7d86384 100644 --- a/test/test_modules/test_model_setup.py +++ b/test/test_modules/test_model_setup.py @@ -1,13 +1,12 @@ -import pytest import os -import keras -import mock -from src.run_modules.model_setup import ModelSetup -from src.run_modules.run_environment import RunEnvironment +import pytest + from src.data_handling.data_generator import DataGenerator -from src.model_modules.model_class import AbstractModelClass from src.datastore import EmptyScope +from src.model_modules.model_class import AbstractModelClass +from src.run_modules.model_setup import ModelSetup +from src.run_modules.run_environment import RunEnvironment class TestModelSetup: diff --git a/test/test_modules/test_post_processing.py b/test/test_modules/test_post_processing.py index e6a245ccb27807d59114d0a7c69e472d98bc1e58..67897451eaa5de065b644d1f9868a846cb57d84e 100644 --- a/test/test_modules/test_post_processing.py +++ b/test/test_modules/test_post_processing.py @@ -1,8 +1,3 @@ -import keras - -from src.run_modules.post_processing import PostProcessing - - class TestPostProcessing: def test_init(self): diff --git a/test/test_modules/test_pre_processing.py b/test/test_modules/test_pre_processing.py index c6f70169adee6eec74c834d952721f41d3c3fa03..9d1feb03ac71b980f6a4cd1b0e6cac2a52d9625b 100644 --- a/test/test_modules/test_pre_processing.py +++ b/test/test_modules/test_pre_processing.py @@ -1,11 +1,12 @@ import logging + import pytest +from src.data_handling.data_generator import DataGenerator +from src.datastore import NameNotFoundInScope from src.helpers import PyTestRegex from src.run_modules.experiment_setup import ExperimentSetup from src.run_modules.pre_processing import PreProcessing, DEFAULT_ARGS_LIST, DEFAULT_KWARGS_LIST -from src.data_handling.data_generator import DataGenerator -from src.datastore import NameNotFoundInScope from src.run_modules.run_environment import RunEnvironment diff --git a/test/test_modules/test_training.py b/test/test_modules/test_training.py index 08b9eaf19e831ed1662efbe21f4ad29d18dff9b4..485348ceca740d8263394fca36efbfbde6dd2d0d 100644 --- a/test/test_modules/test_training.py +++ b/test/test_modules/test_training.py @@ -1,21 +1,22 @@ +import glob +import json +import logging +import os +import shutil + import keras +import mock import pytest from keras.callbacks import ModelCheckpoint, History -import mock -import os -import json -import shutil -import logging -import glob -from src.model_modules.inception_model import InceptionModelBase -from src.model_modules.flatten import flatten_tail -from src.run_modules.training import Training -from src.run_modules.run_environment import RunEnvironment from src.data_handling.data_distributor import Distributor from src.data_handling.data_generator import DataGenerator from src.helpers import PyTestRegex +from src.model_modules.flatten import flatten_tail +from src.model_modules.inception_model import InceptionModelBase from src.model_modules.keras_extensions import LearningRateDecay, HistoryAdvanced +from src.run_modules.run_environment import RunEnvironment +from src.run_modules.training import Training def my_test_model(activation, window_history_size, channels, dropout_rate, add_minor_branch=False): diff --git a/test/test_plotting/test_training_monitoring.py b/test/test_plotting/test_training_monitoring.py index 358a19adf5c81c90f7e77b787ca7b50923990f00..7e4e21c1a28b35bef4aa6e613756378fe41611b5 100644 --- a/test/test_plotting/test_training_monitoring.py +++ b/test/test_plotting/test_training_monitoring.py @@ -1,9 +1,10 @@ +import os + import keras import pytest -import os -from src.plotting.training_monitoring import PlotModelLearningRate, PlotModelHistory from src.model_modules.keras_extensions import LearningRateDecay +from src.plotting.training_monitoring import PlotModelLearningRate, PlotModelHistory @pytest.fixture diff --git a/test/test_statistics.py b/test/test_statistics.py index fa7de6022f0957031f90fbf951679594583eccab..308ac655787e69f90b45e65e7e7df8f35875f652 100644 --- a/test/test_statistics.py +++ b/test/test_statistics.py @@ -1,7 +1,8 @@ +import numpy as np +import pandas as pd import pytest import xarray as xr -import pandas as pd -import numpy as np + from src.statistics import standardise, standardise_inverse, centre, centre_inverse