Skip to content
Snippets Groups Projects
Commit cfce0c9c authored by lukas leufen's avatar lukas leufen
Browse files

Merge branch 'lukas_issue362_feat_branched-rnn' into...

Merge branch 'lukas_issue362_feat_branched-rnn' into 'lukas_issue359_feat_separate-uncertainty-estimate-for-each-forecast-step'

Lukas issue362 feat branched rnn

See merge request !402
parents 6140a4b6 63d15927
No related branches found
No related tags found
5 merge requests!430update recent developments,!413update release branch,!412Resolve "release v2.0.0",!402Lukas issue362 feat branched rnn,!384Resolve "separate uncertainty estimate for each forecast step"
Pipeline #93528 passed
......@@ -44,7 +44,8 @@ class BranchedInputRNN(RNN): # pragma: no cover
for layer, n_hidden in enumerate(conf):
return_sequences = (layer < len(conf) - 1)
x_in_b = self.RNN(n_hidden, return_sequences=return_sequences, recurrent_dropout=self.dropout_rnn,
name=f"{self.RNN.__name__}_branch{branch + 1}_{layer + 1}")(x_in_b)
name=f"{self.RNN.__name__}_branch{branch + 1}_{layer + 1}",
kernel_regularizer=self.kernel_regularizer)(x_in_b)
if self.bn is True:
x_in_b = keras.layers.BatchNormalization()(x_in_b)
x_in_b = self.activation_rnn(name=f"{self.activation_rnn_name}_branch{branch + 1}_{layer + 1}")(x_in_b)
......
......@@ -2,6 +2,7 @@ __author__ = "Lukas Leufen"
__date__ = '2021-05-25'
from functools import reduce, partial
from typing import Union
from mlair.model_modules import AbstractModelClass
from mlair.helpers import select_from_dict
......@@ -33,7 +34,8 @@ class RNN(AbstractModelClass): # pragma: no cover
def __init__(self, input_shape: list, output_shape: list, activation="relu", activation_output="linear",
activation_rnn="tanh", dropout_rnn=0,
optimizer="adam", n_layer=1, n_hidden=10, regularizer=None, dropout=None, layer_configuration=None,
batch_normalization=False, rnn_type="lstm", add_dense_layer=False, dense_layer_configuration=None, **kwargs):
batch_normalization=False, rnn_type="lstm", add_dense_layer=False, dense_layer_configuration=None,
kernel_regularizer=None, **kwargs):
"""
Sets model and loss depending on the given arguments.
......@@ -42,10 +44,12 @@ class RNN(AbstractModelClass): # pragma: no cover
Customize this RNN model via the following parameters:
:param activation: set your desired activation function for appended dense layers (add_dense_layer=True=. Choose
:param activation: set your desired activation function for appended dense layers (add_dense_layer=True). Choose
from relu, tanh, sigmoid, linear, selu, prelu, leakyrelu. (Default relu)
:param activation_rnn: set your desired activation function of the rnn output. Choose from relu, tanh, sigmoid,
linear, selu, prelu, leakyrelu. (Default tanh)
linear, selu, prelu, leakyrelu. To use the fast cuDNN implementation, tensorflow requires to use tanh as
activation. Note that this is not the recurrent activation (which is not mutable in this class) but the
activation of the cell. (Default tanh)
:param activation_output: same as activation parameter but exclusively applied on output layer only. (Default
linear)
:param optimizer: set optimizer method. Can be either adam or sgd. (Default adam)
......@@ -58,7 +62,8 @@ class RNN(AbstractModelClass): # pragma: no cover
:param dropout: use dropout with given rate. If no value is provided, dropout layers are not added to the
network at all. (Default None)
:param dropout_rnn: use recurrent dropout with given rate. This is applied along the recursion and not after
a rnn layer. (Default 0)
a rnn layer. Be aware that tensorflow is only able to use the fast cuDNN implementation with no recurrent
dropout. (Default 0)
:param batch_normalization: use batch normalization layer in the network if enabled. These layers are inserted
between the linear part of a layer (the nn part) and the non-linear part (activation function). No BN layer
is added if set to false. (Default false)
......@@ -94,7 +99,7 @@ class RNN(AbstractModelClass): # pragma: no cover
self.RNN = self._rnn.get(rnn_type.lower())
self._update_model_name(rnn_type)
self.kernel_initializer = self._initializer.get(activation, "glorot_uniform")
# self.kernel_regularizer = self._set_regularizer(regularizer, **kwargs)
self.kernel_regularizer, self.kernel_regularizer_opts = self._set_regularizer(kernel_regularizer, **kwargs)
self.dropout, self.dropout_rate = self._set_dropout(activation, dropout)
assert 0 <= dropout_rnn <= 1
self.dropout_rnn = dropout_rnn
......@@ -121,7 +126,8 @@ class RNN(AbstractModelClass): # pragma: no cover
for layer, n_hidden in enumerate(conf):
return_sequences = (layer < len(conf) - 1)
x_in = self.RNN(n_hidden, return_sequences=return_sequences, recurrent_dropout=self.dropout_rnn)(x_in)
x_in = self.RNN(n_hidden, return_sequences=return_sequences, recurrent_dropout=self.dropout_rnn,
kernel_regularizer=self.kernel_regularizer)(x_in)
if self.bn is True:
x_in = keras.layers.BatchNormalization()(x_in)
x_in = self.activation_rnn(name=f"{self.activation_rnn_name}_{layer + 1}")(x_in)
......@@ -188,23 +194,23 @@ class RNN(AbstractModelClass): # pragma: no cover
return opt(**opt_kwargs)
except KeyError:
raise AttributeError(f"Given optimizer {optimizer} is not supported in this model class.")
#
# def _set_regularizer(self, regularizer, **kwargs):
# if regularizer is None or (isinstance(regularizer, str) and regularizer.lower() == "none"):
# return None
# try:
# reg_name = regularizer.lower()
# reg = self._regularizer.get(reg_name)
# reg_kwargs = {}
# if reg_name in ["l1", "l2"]:
# reg_kwargs = select_from_dict(kwargs, reg_name, remove_none=True)
# if reg_name in reg_kwargs:
# reg_kwargs["l"] = reg_kwargs.pop(reg_name)
# elif reg_name == "l1_l2":
# reg_kwargs = select_from_dict(kwargs, ["l1", "l2"], remove_none=True)
# return reg(**reg_kwargs)
# except KeyError:
# raise AttributeError(f"Given regularizer {regularizer} is not supported in this model class.")
def _set_regularizer(self, regularizer: Union[None, str], **kwargs):
if regularizer is None or (isinstance(regularizer, str) and regularizer.lower() == "none"):
return None, None
try:
reg_name = regularizer.lower()
reg = self._regularizer.get(reg_name)
reg_kwargs = {}
if reg_name in ["l1", "l2"]:
reg_kwargs = select_from_dict(kwargs, reg_name, remove_none=True)
if reg_name in reg_kwargs:
reg_kwargs["l"] = reg_kwargs.pop(reg_name)
elif reg_name == "l1_l2":
reg_kwargs = select_from_dict(kwargs, ["l1", "l2"], remove_none=True)
return reg(**reg_kwargs), reg_kwargs
except KeyError:
raise AttributeError(f"Given regularizer {regularizer} is not supported in this model class.")
def _update_model_name(self, rnn_type):
n_input = str(reduce(lambda x, y: x * y, self._input_shape))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment