From 33940965f7812decca45b5e23ebafcaaff243d10 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Fri, 12 Mar 2021 12:02:20 +0100 Subject: [PATCH 01/12] first CNN class try --- mlair/model_modules/convolutional_networks.py | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 mlair/model_modules/convolutional_networks.py diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py new file mode 100644 index 00000000..f9acdb72 --- /dev/null +++ b/mlair/model_modules/convolutional_networks.py @@ -0,0 +1,113 @@ +__author__ = "Lukas Leufen" +__date__ = '2021-02-' + +from functools import reduce, partial + +from mlair.model_modules import AbstractModelClass +from mlair.helpers import select_from_dict +from mlair.model_modules.loss import var_loss, custom_loss +from mlair.model_modules.advanced_paddings import PadUtils, Padding2D, SymmetricPadding2D + +import keras + + +class CNN(AbstractModelClass): + _activation = {"relu": keras.layers.ReLU, "tanh": partial(keras.layers.Activation, "tanh"), + "sigmoid": partial(keras.layers.Activation, "sigmoid"), + "linear": partial(keras.layers.Activation, "linear"), + "selu": partial(keras.layers.Activation, "selu")} + _initializer = {"selu": keras.initializers.lecun_normal()} + _optimizer = {"adam": keras.optimizers.adam} + _regularizer = {"l1": keras.regularizers.l1, "l2": keras.regularizers.l2, "l1_l2": keras.regularizers.l1_l2} + _requirements = ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad"] + + def __init__(self, input_shape: list, output_shape: list, activation="relu", activation_output="linear", + optimizer="adam", regularizer=None, **kwargs): + + assert len(input_shape) == 1 + assert len(output_shape) == 1 + super().__init__(input_shape[0], output_shape[0]) + + # settings + self.activation = self._set_activation(activation) + self.activation_name = activation + self.activation_output = self._set_activation(activation_output) + self.activation_output_name = activation_output + self.kernel_initializer = self._initializer.get(activation, "glorot_uniform") + self.kernel_regularizer = self._set_regularizer(regularizer, **kwargs) + self.optimizer = self._set_optimizer(optimizer, **kwargs) + + # apply to model + self.set_model() + self.set_compile_options() + self.set_custom_objects(loss=custom_loss([keras.losses.mean_squared_error, var_loss]), var_loss=var_loss) + + def _set_activation(self, activation): + try: + return self._activation.get(activation.lower()) + except KeyError: + raise AttributeError(f"Given activation {activation} is not supported in this model class.") + + def _set_optimizer(self, optimizer, **kwargs): + try: + opt_name = optimizer.lower() + opt = self._optimizer.get(opt_name) + opt_kwargs = {} + if opt_name == "adam": + opt_kwargs = select_from_dict(kwargs, ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad"]) + return opt(**opt_kwargs) + except KeyError: + raise AttributeError(f"Given optimizer {optimizer} is not supported in this model class.") + + def _set_regularizer(self, regularizer, **kwargs): + if regularizer is None or (isinstance(regularizer, str) and regularizer.lower() == "none"): + return None + try: + reg_name = regularizer.lower() + reg = self._regularizer.get(reg_name) + reg_kwargs = {} + if reg_name in ["l1", "l2"]: + reg_kwargs = select_from_dict(kwargs, reg_name, remove_none=True) + if reg_name in reg_kwargs: + reg_kwargs["l"] = reg_kwargs.pop(reg_name) + elif reg_name == "l1_l2": + reg_kwargs = select_from_dict(kwargs, ["l1", "l2"], remove_none=True) + return reg(**reg_kwargs) + except KeyError: + raise AttributeError(f"Given regularizer {regularizer} is not supported in this model class.") + + def set_model(self): + """ + Build the model. + """ + x_input = keras.layers.Input(shape=self._input_shape) + kernel = (1, 1) + pad_size = PadUtils.get_padding_for_same(kernel) + x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad")(x_input) + x_in = keras.layers.Conv2D(filters=16, kernel_size=kernel, + kernel_initializer=self.kernel_initializer, + kernel_regularizer=self.kernel_regularizer)(x_in) + x_in = self.activation()(x_in) + x_in = keras.layers.Conv2D(filters=32, kernel_size=kernel, + kernel_initializer=self.kernel_initializer, + kernel_regularizer=self.kernel_regularizer)(x_in) + x_in = self.activation()(x_in) + x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad")(x_in) + x_in = keras.layers.Conv2D(filters=64, kernel_size=kernel, + kernel_initializer=self.kernel_initializer, + kernel_regularizer=self.kernel_regularizer)(x_in) + x_in = self.activation()(x_in) + x_in = keras.layers.Flatten()(x_in) + x_in = keras.layers.Dense(64, kernel_initializer=self.kernel_initializer, + kernel_regularizer=self.kernel_regularizer)(x_in) + x_in = self.activation()(x_in) + x_in = keras.layers.Dense(16, kernel_initializer=self.kernel_initializer, + kernel_regularizer=self.kernel_regularizer)(x_in) + x_in = self.activation()(x_in) + x_in = keras.layers.Dense(self._output_shape)(x_in) + out = self.activation_output(name=f"{self.activation_output_name}_output")(x_in) + self.model = keras.Model(inputs=x_input, outputs=[out]) + + def set_compile_options(self): + self.compile_options = {"loss": [custom_loss([keras.losses.mean_squared_error, var_loss])], + "metrics": ["mse", "mae", var_loss]} -- GitLab From 30c27e99c0daf6ce0620745c23258c5da18450f8 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Fri, 12 Mar 2021 12:15:23 +0100 Subject: [PATCH 02/12] new pad layer names --- mlair/model_modules/convolutional_networks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py index f9acdb72..e7d1da23 100644 --- a/mlair/model_modules/convolutional_networks.py +++ b/mlair/model_modules/convolutional_networks.py @@ -83,7 +83,7 @@ class CNN(AbstractModelClass): x_input = keras.layers.Input(shape=self._input_shape) kernel = (1, 1) pad_size = PadUtils.get_padding_for_same(kernel) - x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad")(x_input) + x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad1")(x_input) x_in = keras.layers.Conv2D(filters=16, kernel_size=kernel, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) @@ -92,7 +92,7 @@ class CNN(AbstractModelClass): kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) x_in = self.activation()(x_in) - x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad")(x_in) + x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad2")(x_in) x_in = keras.layers.Conv2D(filters=64, kernel_size=kernel, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) -- GitLab From 7e529068a8f5c7c0010a2410e9e8389d667e4cd9 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Fri, 12 Mar 2021 12:43:19 +0100 Subject: [PATCH 03/12] bigger kernel --- mlair/model_modules/convolutional_networks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py index e7d1da23..0a16be7c 100644 --- a/mlair/model_modules/convolutional_networks.py +++ b/mlair/model_modules/convolutional_networks.py @@ -81,7 +81,7 @@ class CNN(AbstractModelClass): Build the model. """ x_input = keras.layers.Input(shape=self._input_shape) - kernel = (1, 1) + kernel = (5, 1) pad_size = PadUtils.get_padding_for_same(kernel) x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad1")(x_input) x_in = keras.layers.Conv2D(filters=16, kernel_size=kernel, @@ -91,8 +91,8 @@ class CNN(AbstractModelClass): x_in = keras.layers.Conv2D(filters=32, kernel_size=kernel, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) + x_in = keras.layers.MaxPooling2D(kernel, strides=(1, 1), padding='valid')(x_in) x_in = self.activation()(x_in) - x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad2")(x_in) x_in = keras.layers.Conv2D(filters=64, kernel_size=kernel, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) -- GitLab From 7673e830a77c78f499c82909ca51d52e282a3609 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Fri, 12 Mar 2021 12:44:41 +0100 Subject: [PATCH 04/12] kernel size can be set from outside --- mlair/model_modules/convolutional_networks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py index 0a16be7c..5146fe52 100644 --- a/mlair/model_modules/convolutional_networks.py +++ b/mlair/model_modules/convolutional_networks.py @@ -22,7 +22,7 @@ class CNN(AbstractModelClass): _requirements = ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad"] def __init__(self, input_shape: list, output_shape: list, activation="relu", activation_output="linear", - optimizer="adam", regularizer=None, **kwargs): + optimizer="adam", regularizer=None, kernel_size=1, **kwargs): assert len(input_shape) == 1 assert len(output_shape) == 1 @@ -35,6 +35,7 @@ class CNN(AbstractModelClass): self.activation_output_name = activation_output self.kernel_initializer = self._initializer.get(activation, "glorot_uniform") self.kernel_regularizer = self._set_regularizer(regularizer, **kwargs) + self.kernel_size = kernel_size self.optimizer = self._set_optimizer(optimizer, **kwargs) # apply to model @@ -81,7 +82,7 @@ class CNN(AbstractModelClass): Build the model. """ x_input = keras.layers.Input(shape=self._input_shape) - kernel = (5, 1) + kernel = (self.kernel_size, 1) pad_size = PadUtils.get_padding_for_same(kernel) x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad1")(x_input) x_in = keras.layers.Conv2D(filters=16, kernel_size=kernel, -- GitLab From 452b590c4b36008a70d1831da01874b9d4d90ac8 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Fri, 12 Mar 2021 14:47:18 +0100 Subject: [PATCH 05/12] no sympad for CNN --- mlair/model_modules/convolutional_networks.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py index 5146fe52..2d8fd9e2 100644 --- a/mlair/model_modules/convolutional_networks.py +++ b/mlair/model_modules/convolutional_networks.py @@ -83,17 +83,15 @@ class CNN(AbstractModelClass): """ x_input = keras.layers.Input(shape=self._input_shape) kernel = (self.kernel_size, 1) - pad_size = PadUtils.get_padding_for_same(kernel) - x_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad1")(x_input) x_in = keras.layers.Conv2D(filters=16, kernel_size=kernel, kernel_initializer=self.kernel_initializer, - kernel_regularizer=self.kernel_regularizer)(x_in) + kernel_regularizer=self.kernel_regularizer)(x_input) x_in = self.activation()(x_in) x_in = keras.layers.Conv2D(filters=32, kernel_size=kernel, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) - x_in = keras.layers.MaxPooling2D(kernel, strides=(1, 1), padding='valid')(x_in) x_in = self.activation()(x_in) + x_in = keras.layers.MaxPooling2D(kernel, strides=(1, 1), padding='valid')(x_in) x_in = keras.layers.Conv2D(filters=64, kernel_size=kernel, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) -- GitLab From a9640da66f529382b584ea9c2dabf3ec420d7e34 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Fri, 12 Mar 2021 15:07:38 +0100 Subject: [PATCH 06/12] fix kernelsize for now --- mlair/model_modules/convolutional_networks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py index 2d8fd9e2..329d1952 100644 --- a/mlair/model_modules/convolutional_networks.py +++ b/mlair/model_modules/convolutional_networks.py @@ -83,16 +83,16 @@ class CNN(AbstractModelClass): """ x_input = keras.layers.Input(shape=self._input_shape) kernel = (self.kernel_size, 1) - x_in = keras.layers.Conv2D(filters=16, kernel_size=kernel, + x_in = keras.layers.Conv2D(filters=16, kernel_size=(73, 1), kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_input) x_in = self.activation()(x_in) - x_in = keras.layers.Conv2D(filters=32, kernel_size=kernel, + x_in = keras.layers.Conv2D(filters=32, kernel_size=(49, 1), kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) x_in = self.activation()(x_in) - x_in = keras.layers.MaxPooling2D(kernel, strides=(1, 1), padding='valid')(x_in) - x_in = keras.layers.Conv2D(filters=64, kernel_size=kernel, + x_in = keras.layers.MaxPooling2D((25, 1), strides=(1, 1), padding='valid')(x_in) + x_in = keras.layers.Conv2D(filters=64, kernel_size=(13, 1), kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) x_in = self.activation()(x_in) -- GitLab From 56f3657c1c07eaa2b617ebf5b2d7435d7f97faa7 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Fri, 12 Mar 2021 15:24:49 +0100 Subject: [PATCH 07/12] changed dense layer --- mlair/model_modules/convolutional_networks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py index 329d1952..c4a10990 100644 --- a/mlair/model_modules/convolutional_networks.py +++ b/mlair/model_modules/convolutional_networks.py @@ -82,7 +82,6 @@ class CNN(AbstractModelClass): Build the model. """ x_input = keras.layers.Input(shape=self._input_shape) - kernel = (self.kernel_size, 1) x_in = keras.layers.Conv2D(filters=16, kernel_size=(73, 1), kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_input) @@ -97,10 +96,10 @@ class CNN(AbstractModelClass): kernel_regularizer=self.kernel_regularizer)(x_in) x_in = self.activation()(x_in) x_in = keras.layers.Flatten()(x_in) - x_in = keras.layers.Dense(64, kernel_initializer=self.kernel_initializer, + x_in = keras.layers.Dense(128, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) x_in = self.activation()(x_in) - x_in = keras.layers.Dense(16, kernel_initializer=self.kernel_initializer, + x_in = keras.layers.Dense(32, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) x_in = self.activation()(x_in) x_in = keras.layers.Dense(self._output_shape)(x_in) -- GitLab From 01dc6fb2e6c26bbb03ee2c3d1827c77f24109743 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Fri, 26 Mar 2021 12:45:38 +0100 Subject: [PATCH 08/12] log more model information during model setup stage --- HPC_setup/requirements_HDFML_additionals.txt | 1 + HPC_setup/requirements_JUWELS_additionals.txt | 1 + mlair/run_modules/model_setup.py | 26 ++++++++++++------- requirements.txt | 1 + requirements_gpu.txt | 1 + 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/HPC_setup/requirements_HDFML_additionals.txt b/HPC_setup/requirements_HDFML_additionals.txt index 12e09ccd..7d6163a6 100644 --- a/HPC_setup/requirements_HDFML_additionals.txt +++ b/HPC_setup/requirements_HDFML_additionals.txt @@ -9,6 +9,7 @@ chardet==4.0.0 coverage==5.4 cycler==0.10.0 dask==2021.2.0 +dill==0.3.3 fsspec==0.8.5 gast==0.4.0 grpcio==1.35.0 diff --git a/HPC_setup/requirements_JUWELS_additionals.txt b/HPC_setup/requirements_JUWELS_additionals.txt index 12e09ccd..7d6163a6 100644 --- a/HPC_setup/requirements_JUWELS_additionals.txt +++ b/HPC_setup/requirements_JUWELS_additionals.txt @@ -9,6 +9,7 @@ chardet==4.0.0 coverage==5.4 cycler==0.10.0 dask==2021.2.0 +dill==0.3.3 fsspec==0.8.5 gast==0.4.0 grpcio==1.35.0 diff --git a/mlair/run_modules/model_setup.py b/mlair/run_modules/model_setup.py index 5dd73d50..8fae430f 100644 --- a/mlair/run_modules/model_setup.py +++ b/mlair/run_modules/model_setup.py @@ -6,6 +6,7 @@ __date__ = '2019-12-02' import logging import os import re +from dill.source import getsource import keras import pandas as pd @@ -57,12 +58,12 @@ class ModelSetup(RunEnvironment): super().__init__() self.model = None exp_name = self.data_store.get("experiment_name") - path = self.data_store.get("model_path") + self.path = self.data_store.get("model_path") self.scope = "model" - self.path = os.path.join(path, f"{exp_name}_%s") - self.model_name = self.path % "%s.h5" - self.checkpoint_name = self.path % "model-best.h5" - self.callbacks_name = self.path % "model-best-callbacks-%s.pickle" + path = os.path.join(self.path, f"{exp_name}_%s") + self.model_name = path % "%s.h5" + self.checkpoint_name = path % "model-best.h5" + self.callbacks_name = path % "model-best-callbacks-%s.pickle" self._train_model = self.data_store.get("train_model") self._create_new_model = self.data_store.get("create_new_model") self._run() @@ -167,6 +168,7 @@ class ModelSetup(RunEnvironment): keras.utils.plot_model(self.model, to_file=file_name, show_shapes=True, show_layer_names=True) def report_model(self): + # report model settings model_settings = self.model.get_settings() model_settings.update(self.model.compile_options) model_settings.update(self.model.optimizer.get_config()) @@ -179,17 +181,23 @@ class ModelSetup(RunEnvironment): if "<" in str(v): v = self._clean_name(str(v)) df.loc[k] = str(v) + df.loc["count params"] = str(self.model.count_params()) df.sort_index(inplace=True) column_format = "ll" path = os.path.join(self.data_store.get("experiment_path"), "latex_report") path_config.check_path_and_create(path) - df.to_latex(os.path.join(path, "model_settings.tex"), na_rep='---', column_format=column_format) - df.to_markdown(open(os.path.join(path, "model_settings.md"), mode="w", encoding='utf-8'), - tablefmt="github") + for p in [path, self.path]: # log to `latex_report` and `model` + df.to_latex(os.path.join(p, "model_settings.tex"), na_rep='---', column_format=column_format) + df.to_markdown(open(os.path.join(p, "model_settings.md"), mode="w", encoding='utf-8'), tablefmt="github") + # report model summary to file + with open(os.path.join(self.path, "model_summary.txt"), "w") as fh: + self.model.summary(print_fn=lambda x: fh.write(x + "\n")) + # print model code to file + with open(os.path.join(self.path, "model_code.txt"), "w") as fh: + fh.write(getsource(self.data_store.get("model_class"))) @staticmethod def _clean_name(orig_name: str): mod_name = re.sub(r'^{0}'.format(re.escape("<")), '', orig_name).replace("'", "").split(" ") mod_name = mod_name[1] if any(map(lambda x: x in mod_name[0], ["class", "function", "method"])) else mod_name[0] return mod_name[:-1] if mod_name[-1] == ">" else mod_name - diff --git a/requirements.txt b/requirements.txt index b0a6e7f5..af742fde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ chardet==4.0.0 coverage==5.4 cycler==0.10.0 dask==2021.2.0 +dill==0.3.3 fsspec==0.8.5 gast==0.4.0 grpcio==1.35.0 diff --git a/requirements_gpu.txt b/requirements_gpu.txt index 35fe0d5e..7dd443a4 100644 --- a/requirements_gpu.txt +++ b/requirements_gpu.txt @@ -9,6 +9,7 @@ chardet==4.0.0 coverage==5.4 cycler==0.10.0 dask==2021.2.0 +dill==0.3.3 fsspec==0.8.5 gast==0.4.0 grpcio==1.35.0 -- GitLab From ef3ced197933ea2e7c022b96f41254cbfbdd09e0 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Mon, 29 Mar 2021 12:08:51 +0200 Subject: [PATCH 09/12] added dropout to CNN --- mlair/model_modules/convolutional_networks.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py index c4a10990..d4955d3d 100644 --- a/mlair/model_modules/convolutional_networks.py +++ b/mlair/model_modules/convolutional_networks.py @@ -12,17 +12,22 @@ import keras class CNN(AbstractModelClass): + _activation = {"relu": keras.layers.ReLU, "tanh": partial(keras.layers.Activation, "tanh"), "sigmoid": partial(keras.layers.Activation, "sigmoid"), "linear": partial(keras.layers.Activation, "linear"), - "selu": partial(keras.layers.Activation, "selu")} - _initializer = {"selu": keras.initializers.lecun_normal()} - _optimizer = {"adam": keras.optimizers.adam} + "selu": partial(keras.layers.Activation, "selu"), + "prelu": partial(keras.layers.PReLU, alpha_initializer=keras.initializers.constant(value=0.25))} + _initializer = {"tanh": "glorot_uniform", "sigmoid": "glorot_uniform", "linear": "glorot_uniform", + "relu": keras.initializers.he_normal(), "selu": keras.initializers.lecun_normal(), + "prelu": keras.initializers.he_normal()} + _optimizer = {"adam": keras.optimizers.adam, "sgd": keras.optimizers.SGD} _regularizer = {"l1": keras.regularizers.l1, "l2": keras.regularizers.l2, "l1_l2": keras.regularizers.l1_l2} - _requirements = ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad"] + _requirements = ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad", "momentum", "nesterov", "l1", "l2"] + _dropout = {"selu": keras.layers.AlphaDropout} def __init__(self, input_shape: list, output_shape: list, activation="relu", activation_output="linear", - optimizer="adam", regularizer=None, kernel_size=1, **kwargs): + optimizer="adam", regularizer=None, kernel_size=1, dropout=None, **kwargs): assert len(input_shape) == 1 assert len(output_shape) == 1 @@ -37,6 +42,7 @@ class CNN(AbstractModelClass): self.kernel_regularizer = self._set_regularizer(regularizer, **kwargs) self.kernel_size = kernel_size self.optimizer = self._set_optimizer(optimizer, **kwargs) + self.dropout, self.dropout_rate = self._set_dropout(activation, dropout) # apply to model self.set_model() @@ -56,6 +62,8 @@ class CNN(AbstractModelClass): opt_kwargs = {} if opt_name == "adam": opt_kwargs = select_from_dict(kwargs, ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad"]) + elif opt_name == "sgd": + opt_kwargs = select_from_dict(kwargs, ["lr", "momentum", "decay", "nesterov"]) return opt(**opt_kwargs) except KeyError: raise AttributeError(f"Given optimizer {optimizer} is not supported in this model class.") @@ -77,6 +85,12 @@ class CNN(AbstractModelClass): except KeyError: raise AttributeError(f"Given regularizer {regularizer} is not supported in this model class.") + def _set_dropout(self, activation, dropout_rate): + if dropout_rate is None: + return None, None + assert 0 <= dropout_rate < 1 + return self._dropout.get(activation, keras.layers.Dropout), dropout_rate + def set_model(self): """ Build the model. -- GitLab From 1ea8b24f2b5af03d7d08a5c6f92738cd8f69135c Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Mon, 29 Mar 2021 12:28:15 +0200 Subject: [PATCH 10/12] join module now uses a retry strategy, /close #296 on test success --- mlair/helpers/join.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mlair/helpers/join.py b/mlair/helpers/join.py index 8a8ca0b8..e0b28660 100644 --- a/mlair/helpers/join.py +++ b/mlair/helpers/join.py @@ -8,6 +8,8 @@ from typing import Iterator, Union, List, Dict import pandas as pd import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry from mlair import helpers from mlair.configuration.join_settings import join_settings @@ -129,13 +131,24 @@ def get_data(opts: Dict, headers: Dict) -> Union[Dict, List]: :return: requested data (either as list or dictionary) """ url = create_url(**opts) - response = requests.get(url, headers=headers) + response = retries_session().get(url, headers=headers) if response.status_code == 200: return response.json() else: raise EmptyQueryResult(f"There was an error (STATUS {response.status_code}) for request {url}") +def retries_session(max_retries=5): + retry_strategy = Retry(total=max_retries, + status_forcelist=[429, 500, 502, 503, 504], + method_whitelist=["HEAD", "GET", "OPTIONS"]) + adapter = HTTPAdapter(max_retries=retry_strategy) + http = requests.Session() + http.mount("https://", adapter) + http.mount("http://", adapter) + return http + + def load_series_information(station_name: List[str], station_type: str_or_none, network_name: str_or_none, join_url_base: str, headers: Dict, data_origin: Dict = None) -> [Dict, Dict]: """ -- GitLab From 4dae57e381de56f823625f4b48269de6cdbe8f28 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Mon, 29 Mar 2021 14:31:08 +0200 Subject: [PATCH 11/12] use dropout in CNNs --- mlair/model_modules/convolutional_networks.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py index d4955d3d..624cfa09 100644 --- a/mlair/model_modules/convolutional_networks.py +++ b/mlair/model_modules/convolutional_networks.py @@ -104,11 +104,15 @@ class CNN(AbstractModelClass): kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) x_in = self.activation()(x_in) + if self.dropout is not None: + x_in = self.dropout(self.dropout_rate)(x_in) x_in = keras.layers.MaxPooling2D((25, 1), strides=(1, 1), padding='valid')(x_in) x_in = keras.layers.Conv2D(filters=64, kernel_size=(13, 1), kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) x_in = self.activation()(x_in) + if self.dropout is not None: + x_in = self.dropout(self.dropout_rate)(x_in) x_in = keras.layers.Flatten()(x_in) x_in = keras.layers.Dense(128, kernel_initializer=self.kernel_initializer, kernel_regularizer=self.kernel_regularizer)(x_in) -- GitLab From 4a1d7679686026da0755dbdf6d2f492a57f13c52 Mon Sep 17 00:00:00 2001 From: leufen1 <l.leufen@fz-juelich.de> Date: Mon, 29 Mar 2021 14:57:11 +0200 Subject: [PATCH 12/12] fix for #296 to reduce waiting if not internet connection could be established --- mlair/helpers/join.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mlair/helpers/join.py b/mlair/helpers/join.py index e0b28660..93cb0e7b 100644 --- a/mlair/helpers/join.py +++ b/mlair/helpers/join.py @@ -131,15 +131,16 @@ def get_data(opts: Dict, headers: Dict) -> Union[Dict, List]: :return: requested data (either as list or dictionary) """ url = create_url(**opts) - response = retries_session().get(url, headers=headers) + response = retries_session().get(url, headers=headers, timeout=(5, None)) # timeout=(open, read) if response.status_code == 200: return response.json() else: raise EmptyQueryResult(f"There was an error (STATUS {response.status_code}) for request {url}") -def retries_session(max_retries=5): +def retries_session(max_retries=3): retry_strategy = Retry(total=max_retries, + backoff_factor=0.1, status_forcelist=[429, 500, 502, 503, 504], method_whitelist=["HEAD", "GET", "OPTIONS"]) adapter = HTTPAdapter(max_retries=retry_strategy) -- GitLab