From ac14f241bf8427a40c3420a1b3d6cecf636e0c3b Mon Sep 17 00:00:00 2001
From: Felix Kleinert <f.kleinert@fz-juelich.de>
Date: Tue, 30 Nov 2021 13:41:52 +0100
Subject: [PATCH] FK: prepare merge with dev
---
mlair/model_modules/model_class.py | 40 +++-------------
mlair/plotting/postprocessing_plotting.py | 58 ++++++++++++-----------
2 files changed, 37 insertions(+), 61 deletions(-)
diff --git a/mlair/model_modules/model_class.py b/mlair/model_modules/model_class.py
index 96cfdccf..fd2d530d 100644
--- a/mlair/model_modules/model_class.py
+++ b/mlair/model_modules/model_class.py
@@ -586,8 +586,8 @@ class MySimpleConv1D(AbstractModelClass):
)
self.loss = keras.losses.mean_squared_error
self.compile_options = {"metrics": ["mse", "mae"]}
-
-
+
+
class MyUnet(AbstractModelClass):
"""
Original source code from https://www.kaggle.com/keegil/keras-u-net-starter-lb-0-277
@@ -596,7 +596,7 @@ class MyUnet(AbstractModelClass):
def __init__(self, input_shape: list, output_shape: list):
super().__init__(input_shape[0], output_shape[0])
- self.first_filter_size = self._input_shape[-1] # 16
+ self.first_filter_size = 16*2#self._input_shape[-1] # 16
self.lstm_units = 64 * 2
self.kernel_size = (3, 1) # (3,1)
self.activation = "elu"
@@ -671,34 +671,6 @@ class MyUnet(AbstractModelClass):
ls2 = keras.layers.LSTM(self.lstm_units, return_sequences=True)(ls2)
c4_2 = keras.layers.Reshape((c3.shape[1], 1, -1))(ls2)
- # c4 = Padding2D("SymPad2D")(padding=pad_size)(p3)
- # c4 = keras.layers.Conv2D(128, self.kernel_size, activation=self.activation, kernel_initializer=self.kernel_initializer)(c4)
- # c4 = keras.layers.Dropout(0.2)(c4)
- # c4 = Padding2D("SymPad2D")(padding=pad_size)(c4)
- # c4 = keras.layers.Conv2D(128, self.kernel_size, activation=self.activation, kernel_initializer=self.kernel_initializer, name='c4')(c4)
- # p4 = c4
- # # p4 = keras.layers.MaxPooling2D(self.pool_size)(c4)
-
- # c5 = Padding2D("SymPad2D")(padding=pad_size)(p4)
- # c5 = keras.layers.Conv2D(256, self.kernel_size, activation=self.activation, kernel_initializer=self.kernel_initializer)(c5)
- # c5 = keras.layers.Dropout(0.3)(c5)
- # c5 = Padding2D("SymPad2D")(padding=pad_size)(c5)
- # c5 = keras.layers.Conv2D(256, self.kernel_size, activation=self.activation, kernel_initializer=self.kernel_initializer, name='c5tou6')(c5)
-
- # u6 = c5
- # # u6 = Padding2D("SymPad2D")(padding=pad_size)(c5)
- # # u6 = keras.layers.Conv2DTranspose(128, self.pool_size, strides=self.pool_size)(u6)
- # u6 = keras.layers.concatenate([u6, c4], name="u6_c4")
- # c6 = Padding2D("SymPad2D")(padding=pad_size)(u6)
- # c6 = keras.layers.Conv2D(128, self.kernel_size, activation=self.activation, kernel_initializer=self.kernel_initializer)(c6)
- # c6 = keras.layers.Dropout(0.2)(c6)
- # c6 = Padding2D("SymPad2D")(padding=pad_size)(c6)
- # c6 = keras.layers.Conv2D(128, self.kernel_size, activation=self.activation, kernel_initializer=self.kernel_initializer, name='c6tou7')(c6)
-
- # u7 = c6
- # u7 = Padding2D("SymPad2D")(padding=pad_size)(c6)
- # u7 = keras.layers.Conv2DTranspose(64, self.pool_size, strides=self.pool_size)(u7)
-
u7 = keras.layers.UpSampling2D(size=(3, 1))(c4)
cn3 = Padding2D("SymPad2D")(padding=pad_size)(c3)
# u7 = c4
@@ -758,8 +730,10 @@ class MyUnet(AbstractModelClass):
self.optimizer = keras.optimizers.Adam(lr=self.initial_lr,
# clipnorm=self.clipnorm,
)
- self.loss = keras.losses.mean_squared_error
- self.compile_options = {"metrics": ["mse", "mae"]}
+
+ loss = keras.losses.mean_squared_error
+ self.compile_options = {"loss": [loss], "metrics": ["mse", "mae"]}
+
class MySimpleConv2D(AbstractModelClass):
diff --git a/mlair/plotting/postprocessing_plotting.py b/mlair/plotting/postprocessing_plotting.py
index 4a819605..29b609fd 100644
--- a/mlair/plotting/postprocessing_plotting.py
+++ b/mlair/plotting/postprocessing_plotting.py
@@ -7,6 +7,7 @@ import math
import os
import warnings
from typing import Dict, List, Tuple
+import sys
import matplotlib
import matplotlib.pyplot as plt
@@ -309,13 +310,31 @@ class PlotConditionalQuantiles(AbstractPlotClass): # pragma: no cover
self._plot_seasons()
self._plot_all()
+ #def _plot_seasons(self):
+ # """Create seasonal plots."""
+ # for season in self._seasons:
+ # self._plot_base(data=self._data.where(self._data['index.season'] == season), x_model=self._model_name,
+ # y_model=self._obs_name, plot_name_affix="cali-ref", season=season)
+ # self._plot_base(data=self._data.where(self._data['index.season'] == season), x_model=self._obs_name,
+ # y_model=self._model_name, plot_name_affix="like-base", season=season)
+
def _plot_seasons(self):
"""Create seasonal plots."""
for season in self._seasons:
- self._plot_base(data=self._data.where(self._data['index.season'] == season), x_model=self._model_name,
- y_model=self._obs_name, plot_name_affix="cali-ref", season=season)
- self._plot_base(data=self._data.where(self._data['index.season'] == season), x_model=self._obs_name,
- y_model=self._model_name, plot_name_affix="like-base", season=season)
+ try:
+ self._plot_base(data=self._data.where(self._data['index.season'] == season), x_model=self._model_name,
+ y_model=self._obs_name, plot_name_affix="cali-ref", season=season)
+ except Exception as e:
+ logging.error(f"Could not create plot PlotConditionalQuantiles._plot_seasons: {season}, cali-ref"
+ f" due to the following error:"
+ f"\n{sys.exc_info()[0]}\n{sys.exc_info()[1]}\n{sys.exc_info()[2]}")
+ try:
+ self._plot_base(data=self._data.where(self._data['index.season'] == season), x_model=self._obs_name,
+ y_model=self._model_name, plot_name_affix="like-base", season=season)
+ except Exception as e:
+ logging.error(f"Could not create plot PlotConditionalQuantiles._plot_seasons: {season}, like-base"
+ f" due to the following error:"
+ f"\n{sys.exc_info()[0]}\n{sys.exc_info()[1]}\n{sys.exc_info()[2]}")
def _plot_all(self):
"""Plot overall conditional quantiles on full data."""
@@ -489,27 +508,15 @@ class PlotCompetitiveSkillScore(AbstractPlotClass): # pragma: no cover
calculated comparisons for cnn, persistence and ols.
:param plot_folder: path to save the plot (default: current directory)
:param model_setup: architecture type (default "CNN")
- :param sampling: sampling of data (default "daily") used for legend
"""
-#<<<<<<< HEAD
- def __init__(self, data: pd.DataFrame, plot_folder=".", model_setup="NN", sampling="daily",
- model_name_for_plots=None):
-#=======
-# def __init__(self, data: Dict[str, pd.DataFrame], plot_folder=".", model_setup="NN"):
-#>>>>>>> develop
+ def __init__(self, data: Dict[str, pd.DataFrame], plot_folder=".", model_setup="NN", **kwargs):
"""Initialise."""
super().__init__(plot_folder, f"skill_score_competitive_{model_setup}")
self._model_setup = model_setup
- self._sampling = self._get_sampling(sampling)
self._labels = None
-#<<<<<<< HEAD
- self._model_name_for_plots = model_name_for_plots
- self._data = self._prepare_data(data)
-#=======
-# self._data = self._prepare_data(helpers.remove_items(data, "total"))
-#>>>>>>> develop
+ self._data = self._prepare_data(helpers.remove_items(data, "total"))
default_plot_name = self.plot_name
# draw full detail plot
self.plot_name = default_plot_name + "_full_detail"
@@ -541,7 +548,7 @@ class PlotCompetitiveSkillScore(AbstractPlotClass): # pragma: no cover
data = data.rename({"stations_level_0": "stations", "stations_level_1": "comparison"})
data = data.to_dataframe("data").unstack(level=1).swaplevel()
data.columns = data.columns.levels[1]
- self._labels = [str(i) + self._sampling for i in data.index.levels[1].values]
+ self._labels = [str(i) + "d" for i in data.index.levels[1].values]
data = data.stack(level=0).reset_index(level=2, drop=True).reset_index(name="data")
return data.astype({"comparison": str, "ahead": int, "data": float})
@@ -550,16 +557,14 @@ class PlotCompetitiveSkillScore(AbstractPlotClass): # pragma: no cover
size = max([len(np.unique(self._data.comparison)), 6])
fig, ax = plt.subplots(figsize=(size, size * 0.8))
data = self._filter_comparisons(self._data) if single_model_comparison is True else self._data
- if self._model_name_for_plots is not None:
- data.loc[:, 'comparison'] = [i.replace('nn-', f'{self._model_name_for_plots}-') for i in data['comparison']]
order = self._create_pseudo_order(data)
- sns.boxplot(x="comparison", y="data", hue="ahead", data=data, whis=1., ax=ax, palette="Blues_d",
+ sns.boxplot(x="comparison", y="data", hue="ahead", data=data, whis=1.5, ax=ax, palette="Blues_d",
showmeans=True, meanprops={"markersize": 3, "markeredgecolor": "k"}, flierprops={"marker": "."},
order=order)
ax.axhline(y=0, color="grey", linewidth=.5)
ax.set(ylabel="skill score", xlabel="competing models", title="summary of all stations", ylim=self._lim(data))
handles, _ = ax.get_legend_handles_labels()
- plt.xticks(rotation=45, horizontalalignment="right")
+ plt.xticks(rotation=90)
ax.legend(handles, self._labels)
plt.tight_layout()
@@ -567,10 +572,8 @@ class PlotCompetitiveSkillScore(AbstractPlotClass): # pragma: no cover
"""Plot skill scores of the comparisons, but vertically aligned."""
fig, ax = plt.subplots()
data = self._filter_comparisons(self._data) if single_model_comparison is True else self._data
- if self._model_name_for_plots is not None:
- data.loc[:, 'comparison'] = [i.replace('nn-', f'{self._model_name_for_plots}-') for i in data['comparison']]
order = self._create_pseudo_order(data)
- sns.boxplot(y="comparison", x="data", hue="ahead", data=data, whis=1., ax=ax, palette="Blues_d",
+ sns.boxplot(y="comparison", x="data", hue="ahead", data=data, whis=1.5, ax=ax, palette="Blues_d",
showmeans=True, meanprops={"markersize": 3, "markeredgecolor": "k"}, flierprops={"marker": "."},
order=order)
ax.axvline(x=0, color="grey", linewidth=.5)
@@ -587,8 +590,7 @@ class PlotCompetitiveSkillScore(AbstractPlotClass): # pragma: no cover
return uniq[index.argsort()]
def _filter_comparisons(self, data):
- # filtered_headers = list(filter(lambda x: "nn-" in x, data.comparison.unique()))
- filtered_headers = list(filter(lambda x: f"{self._model_name_for_plots}-" in x, data.comparison.unique()))
+ filtered_headers = list(filter(lambda x: "nn-" in x, data.comparison.unique()))
return data[data.comparison.isin(filtered_headers)]
@staticmethod
--
GitLab