From 9b27fd76f65beb5a7f85e98a8a0b6ef796e63885 Mon Sep 17 00:00:00 2001
From: Falco Weichselbaum <f.weichselbaum@fz-juelich.de>
Date: Tue, 12 Oct 2021 17:07:42 +0200
Subject: [PATCH] tf_upgrade_tf2 run on whole directory. Only training.py and
abstract_model_class.py directly affected by the script. Other problems were
mostly import keras related and solved with import tensorflow.keras as keras.
Not possible for advanced_paddings.py. For details see:
https://gitlab.jsc.fz-juelich.de/esde/machine-learning/mlair/-/issues/331
---
mlair/data_handler/iterator.py | 2 +-
mlair/model_modules/abstract_model_class.py | 6 +-
mlair/model_modules/advanced_paddings.py | 16 ++---
mlair/model_modules/convolutional_networks.py | 2 +-
.../model_modules/fully_connected_networks.py | 6 +-
mlair/model_modules/inception_model.py | 4 +-
mlair/model_modules/keras_extensions.py | 4 +-
mlair/model_modules/loss.py | 2 +-
mlair/model_modules/recurrent_networks.py | 4 +-
mlair/plotting/training_monitoring.py | 2 +-
mlair/run_modules/model_setup.py | 2 +-
mlair/run_modules/post_processing.py | 2 +-
mlair/run_modules/training.py | 6 +-
requirements_tf2.txt | 69 +++++++++++++++++++
run.py | 4 +-
run_climate_filter.py | 0
16 files changed, 101 insertions(+), 30 deletions(-)
create mode 100644 requirements_tf2.txt
mode change 100755 => 100644 run_climate_filter.py
diff --git a/mlair/data_handler/iterator.py b/mlair/data_handler/iterator.py
index 564bf3bf..f2e3b689 100644
--- a/mlair/data_handler/iterator.py
+++ b/mlair/data_handler/iterator.py
@@ -3,7 +3,7 @@ __author__ = 'Lukas Leufen'
__date__ = '2020-07-07'
from collections import Iterator, Iterable
-import keras
+import tensorflow.keras as keras
import numpy as np
import math
import os
diff --git a/mlair/model_modules/abstract_model_class.py b/mlair/model_modules/abstract_model_class.py
index 989f4578..692c7d8c 100644
--- a/mlair/model_modules/abstract_model_class.py
+++ b/mlair/model_modules/abstract_model_class.py
@@ -2,7 +2,7 @@ import inspect
from abc import ABC
from typing import Any, Dict, Callable
-import keras
+import tensorflow.keras as keras
import tensorflow as tf
from mlair.helpers import remove_items
@@ -173,8 +173,8 @@ class AbstractModelClass(ABC):
"""
if first.__class__ == second.__class__ and first.__module__ == 'keras.optimizers':
res = True
- init = tf.global_variables_initializer()
- with tf.Session() as sess:
+ init = tf.compat.v1.global_variables_initializer()
+ with tf.compat.v1.Session() as sess:
sess.run(init)
for k, v in first.__dict__.items():
try:
diff --git a/mlair/model_modules/advanced_paddings.py b/mlair/model_modules/advanced_paddings.py
index f2fd4de9..e30002e7 100644
--- a/mlair/model_modules/advanced_paddings.py
+++ b/mlair/model_modules/advanced_paddings.py
@@ -8,12 +8,12 @@ from typing import Union, Tuple
import numpy as np
import tensorflow as tf
-from keras.backend.common import normalize_data_format
-from keras.layers import ZeroPadding2D
-from keras.layers.convolutional import _ZeroPadding
-from keras.legacy import interfaces
-from keras.utils import conv_utils
-from keras.utils.generic_utils import transpose_shape
+from tensorflow.keras.backend.common import normalize_data_format
+from tensorflow.keras.layers import ZeroPadding2D
+from tensorflow.keras.layers.convolutional import _ZeroPadding
+from tensorflow.keras.legacy import interfaces
+from tensorflow.keras.utils import conv_utils
+from tensorflow.keras.utils.generic_utils import transpose_shape
class PadUtils:
@@ -190,7 +190,7 @@ class ReflectionPadding2D(_ZeroPadding):
def call(self, inputs, mask=None):
"""Call ReflectionPadding2D."""
pattern = PadUtils.spatial_2d_padding(padding=self.padding, data_format=self.data_format)
- return tf.pad(inputs, pattern, 'REFLECT')
+ return tf.pad(tensor=inputs, paddings=pattern, mode='REFLECT')
class SymmetricPadding2D(_ZeroPadding):
@@ -264,7 +264,7 @@ class SymmetricPadding2D(_ZeroPadding):
def call(self, inputs, mask=None):
"""Call SymmetricPadding2D."""
pattern = PadUtils.spatial_2d_padding(padding=self.padding, data_format=self.data_format)
- return tf.pad(inputs, pattern, 'SYMMETRIC')
+ return tf.pad(tensor=inputs, paddings=pattern, mode='SYMMETRIC')
class Padding2D:
diff --git a/mlair/model_modules/convolutional_networks.py b/mlair/model_modules/convolutional_networks.py
index 624cfa09..294e31af 100644
--- a/mlair/model_modules/convolutional_networks.py
+++ b/mlair/model_modules/convolutional_networks.py
@@ -8,7 +8,7 @@ from mlair.helpers import select_from_dict
from mlair.model_modules.loss import var_loss, custom_loss
from mlair.model_modules.advanced_paddings import PadUtils, Padding2D, SymmetricPadding2D
-import keras
+import tensorflow.keras as keras
class CNN(AbstractModelClass):
diff --git a/mlair/model_modules/fully_connected_networks.py b/mlair/model_modules/fully_connected_networks.py
index 03380333..8536516e 100644
--- a/mlair/model_modules/fully_connected_networks.py
+++ b/mlair/model_modules/fully_connected_networks.py
@@ -7,7 +7,7 @@ from mlair.model_modules import AbstractModelClass
from mlair.helpers import select_from_dict
from mlair.model_modules.loss import var_loss, custom_loss, l_p_loss
-import keras
+import tensorflow.keras as keras
class FCN(AbstractModelClass):
@@ -25,7 +25,7 @@ class FCN(AbstractModelClass):
_initializer = {"tanh": "glorot_uniform", "sigmoid": "glorot_uniform", "linear": "glorot_uniform",
"relu": keras.initializers.he_normal(), "selu": keras.initializers.lecun_normal(),
"prelu": keras.initializers.he_normal()}
- _optimizer = {"adam": keras.optimizers.adam, "sgd": keras.optimizers.SGD}
+ _optimizer = {"adam": keras.optimizers.Adam, "sgd": keras.optimizers.SGD}
_regularizer = {"l1": keras.regularizers.l1, "l2": keras.regularizers.l2, "l1_l2": keras.regularizers.l1_l2}
_requirements = ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad", "momentum", "nesterov", "l1", "l2"]
_dropout = {"selu": keras.layers.AlphaDropout}
@@ -207,7 +207,7 @@ class BranchedInputFCN(AbstractModelClass):
_initializer = {"tanh": "glorot_uniform", "sigmoid": "glorot_uniform", "linear": "glorot_uniform",
"relu": keras.initializers.he_normal(), "selu": keras.initializers.lecun_normal(),
"prelu": keras.initializers.he_normal()}
- _optimizer = {"adam": keras.optimizers.adam, "sgd": keras.optimizers.SGD}
+ _optimizer = {"adam": keras.optimizers.Adam, "sgd": keras.optimizers.SGD}
_regularizer = {"l1": keras.regularizers.l1, "l2": keras.regularizers.l2, "l1_l2": keras.regularizers.l1_l2}
_requirements = ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad", "momentum", "nesterov", "l1", "l2"]
_dropout = {"selu": keras.layers.AlphaDropout}
diff --git a/mlair/model_modules/inception_model.py b/mlair/model_modules/inception_model.py
index d7354c37..0387a5f2 100644
--- a/mlair/model_modules/inception_model.py
+++ b/mlair/model_modules/inception_model.py
@@ -3,8 +3,8 @@ __date__ = '2019-10-22'
import logging
-import keras
-import keras.layers as layers
+import tensorflow.keras as keras
+import tensorflow.keras.layers as layers
from mlair.model_modules.advanced_paddings import PadUtils, ReflectionPadding2D, Padding2D
diff --git a/mlair/model_modules/keras_extensions.py b/mlair/model_modules/keras_extensions.py
index e0f54282..d890e7b0 100644
--- a/mlair/model_modules/keras_extensions.py
+++ b/mlair/model_modules/keras_extensions.py
@@ -11,8 +11,8 @@ from typing_extensions import TypedDict
from time import time
import numpy as np
-from keras import backend as K
-from keras.callbacks import History, ModelCheckpoint, Callback
+from tensorflow.keras import backend as K
+from tensorflow.keras.callbacks import History, ModelCheckpoint, Callback
from mlair import helpers
diff --git a/mlair/model_modules/loss.py b/mlair/model_modules/loss.py
index 2034c5a7..1a54bc1c 100644
--- a/mlair/model_modules/loss.py
+++ b/mlair/model_modules/loss.py
@@ -1,6 +1,6 @@
"""Collection of different customised loss functions."""
-from keras import backend as K
+from tensorflow.keras import backend as K
from typing import Callable
diff --git a/mlair/model_modules/recurrent_networks.py b/mlair/model_modules/recurrent_networks.py
index 95c48bc8..59927e99 100644
--- a/mlair/model_modules/recurrent_networks.py
+++ b/mlair/model_modules/recurrent_networks.py
@@ -7,7 +7,7 @@ from mlair.model_modules import AbstractModelClass
from mlair.helpers import select_from_dict
from mlair.model_modules.loss import var_loss, custom_loss
-import keras
+import tensorflow.keras as keras
class RNN(AbstractModelClass):
@@ -24,7 +24,7 @@ class RNN(AbstractModelClass):
_initializer = {"tanh": "glorot_uniform", "sigmoid": "glorot_uniform", "linear": "glorot_uniform",
"relu": keras.initializers.he_normal(), "selu": keras.initializers.lecun_normal(),
"prelu": keras.initializers.he_normal()}
- _optimizer = {"adam": keras.optimizers.adam, "sgd": keras.optimizers.SGD}
+ _optimizer = {"adam": keras.optimizers.Adam, "sgd": keras.optimizers.SGD}
_regularizer = {"l1": keras.regularizers.l1, "l2": keras.regularizers.l2, "l1_l2": keras.regularizers.l1_l2}
_requirements = ["lr", "beta_1", "beta_2", "epsilon", "decay", "amsgrad", "momentum", "nesterov", "l1", "l2"]
_dropout = {"selu": keras.layers.AlphaDropout}
diff --git a/mlair/plotting/training_monitoring.py b/mlair/plotting/training_monitoring.py
index 9cad9fd0..b2b531b9 100644
--- a/mlair/plotting/training_monitoring.py
+++ b/mlair/plotting/training_monitoring.py
@@ -5,7 +5,7 @@ __date__ = '2019-12-11'
from typing import Union, Dict, List
-import keras
+import tensorflow.keras as keras
import matplotlib
import matplotlib.pyplot as plt
import pandas as pd
diff --git a/mlair/run_modules/model_setup.py b/mlair/run_modules/model_setup.py
index 83f4a2bd..0b9e8ec5 100644
--- a/mlair/run_modules/model_setup.py
+++ b/mlair/run_modules/model_setup.py
@@ -8,7 +8,7 @@ import os
import re
from dill.source import getsource
-import keras
+import tensorflow.keras as keras
import pandas as pd
import tensorflow as tf
diff --git a/mlair/run_modules/post_processing.py b/mlair/run_modules/post_processing.py
index 4387f359..180fb3d2 100644
--- a/mlair/run_modules/post_processing.py
+++ b/mlair/run_modules/post_processing.py
@@ -10,7 +10,7 @@ import sys
import traceback
from typing import Dict, Tuple, Union, List, Callable
-import keras
+import tensorflow.keras as keras
import numpy as np
import pandas as pd
import xarray as xr
diff --git a/mlair/run_modules/training.py b/mlair/run_modules/training.py
index 00e8eae1..62c8d19d 100644
--- a/mlair/run_modules/training.py
+++ b/mlair/run_modules/training.py
@@ -8,8 +8,8 @@ import logging
import os
from typing import Union
-import keras
-from keras.callbacks import Callback, History
+import tensorflow.keras as keras
+from tensorflow.keras.callbacks import Callback, History
import psutil
import pandas as pd
@@ -178,7 +178,7 @@ class Training(RunEnvironment):
"""Save model in local experiment directory. Model is named as `<experiment_name>_<custom_model_name>.h5`."""
model_name = self.data_store.get("model_name", "model")
logging.debug(f"save best model to {model_name}")
- self.model.save(model_name)
+ self.model.save(model_name, save_format='h5')
self.data_store.set("best_model", self.model)
def load_best_model(self, name: str) -> None:
diff --git a/requirements_tf2.txt b/requirements_tf2.txt
new file mode 100644
index 00000000..21cc5f36
--- /dev/null
+++ b/requirements_tf2.txt
@@ -0,0 +1,69 @@
+absl-py==0.11.0
+appdirs==1.4.4
+astor==0.8.1
+astropy==4.1
+attrs==20.3.0
+bottleneck==1.3.2
+cached-property==1.5.2
+certifi==2020.12.5
+cftime==1.4.1
+chardet==4.0.0
+coverage==5.4
+cycler==0.10.0
+dask==2021.2.0
+dill==0.3.3
+fsspec==0.8.5
+gast==0.3.3
+grpcio==1.35.0
+h5py==2.10.0
+idna==2.10
+importlib-metadata==3.4.0
+iniconfig==1.1.1
+kiwisolver==1.3.1
+locket==0.2.1
+Markdown==3.3.3
+matplotlib==3.3.4
+mock==4.0.3
+netCDF4==1.5.5.1
+numpy==1.19.5
+ordered-set==4.0.2
+packaging==20.9
+pandas==1.1.5
+partd==1.1.0
+patsy==0.5.1
+Pillow==8.1.0
+pluggy==0.13.1
+protobuf==3.15.0
+psutil==5.8.0
+py==1.10.0
+pydot==1.4.2
+pyparsing==2.4.7
+pyshp==2.1.3
+pytest==6.2.2
+pytest-cov==2.11.1
+pytest-html==3.1.1
+pytest-lazy-fixture==0.6.3
+pytest-metadata==1.11.0
+pytest-sugar==0.9.4
+python-dateutil==2.8.1
+pytz==2021.1
+PyYAML==5.4.1
+requests==2.25.1
+scipy==1.4.1
+seaborn==0.11.1
+six==1.15.0
+statsmodels==0.12.2
+tabulate==0.8.8
+termcolor==1.1.0
+toml==0.10.2
+toolz==0.11.1
+typing-extensions==3.7.4.3
+urllib3==1.26.3
+Werkzeug==1.0.1
+wget==3.2
+xarray==0.16.2
+zipp==3.4.0
+
+--no-binary shapely Shapely==1.7.0
+# Cartopy==0.18.0
+tensorflow==2.2.0
diff --git a/run.py b/run.py
index eb703e11..fbe6aa26 100644
--- a/run.py
+++ b/run.py
@@ -6,6 +6,7 @@ from mlair.workflows import DefaultWorkflow
from mlair.helpers import remove_items
from mlair.configuration.defaults import DEFAULT_PLOT_LIST
import os
+import tensorflow as tf
def load_stations():
@@ -20,7 +21,8 @@ def load_stations():
def main(parser_args):
- plots = remove_items(DEFAULT_PLOT_LIST, "PlotConditionalQuantiles")
+ tf.compat.v1.disable_v2_behavior()
+ plots = remove_items(DEFAULT_PLOT_LIST, ["PlotConditionalQuantiles", "PlotPeriodogram"])
workflow = DefaultWorkflow( # stations=load_stations(),
# stations=["DEBW087","DEBW013", "DEBW107", "DEBW076"],
stations=["DEBW013", "DEBW087", "DEBW107", "DEBW076"],
diff --git a/run_climate_filter.py b/run_climate_filter.py
old mode 100755
new mode 100644
--
GitLab