Merge branch 'develop' into lukas_issue346_experimental_load-preprocessing-snapshot

# Conflicts:
#	mlair/run_modules/pre_processing.py

.gitlab-ci.yml

@@ -41,12 +41,16 @@ tests (from scratch):
   before_script:
     - chmod +x ./CI/update_badge.sh
     - ./CI/update_badge.sh > /dev/null
+    - source /opt/venv/bin/activate
   script:
     - pip install --upgrade pip
-    - pip install numpy wheel six==1.15.0
-    - zypper --non-interactive install binutils libproj-devel gdal-devel
-    - zypper --non-interactive install proj geos-devel
-    #    - cat requirements.txt | cut -f1 -d"#" | sed '/^\s*$/d' | xargs -L 1 pip install
+    - zypper --no-gpg-checks addrepo https://download.opensuse.org/repositories/Application:Geo/15.4/Application:Geo.repo
+    - zypper --no-gpg-checks refresh
+    - zypper --no-gpg-checks --non-interactive install proj=8.2.1
+    - zypper --no-gpg-checks --non-interactive install geos=3.10.3
+    - zypper --no-gpg-checks --non-interactive install geos-devel=3.9.1
+    - zypper --no-gpg-checks --non-interactive install libproj22=8.2.1
+    - zypper --no-gpg-checks --non-interactive install binutils libproj-devel gdal-devel
     - pip install -r requirements.txt
     - chmod +x ./CI/run_pytest.sh
     - ./CI/run_pytest.sh
@@ -60,34 +64,6 @@ tests (from scratch):
       - badges/
       - test_results/
 
-### Tests (on GPU) ###
-#tests (on GPU):
-#  tags:
-#    - gpu
-#    - zam347
-#  stage: test
-#  only:
-#    - master
-#    - /^release.*$/
-#    - develop
-#  variables:
-#    FAILURE_THRESHOLD: 100
-#    TEST_TYPE: "gpu"
-#  before_script:
-#    - chmod +x ./CI/update_badge.sh
-#    - ./CI/update_badge.sh > /dev/null
-#  script:
-#    - pip install -r test/requirements_tf_skip.txt
-#    - chmod +x ./CI/run_pytest.sh
-#    - ./CI/run_pytest.sh
-#  after_script:
-#    - ./CI/update_badge.sh > /dev/null
-#  artifacts:
-#    name: pages
-#    when: always
-#    paths:
-#      - badges/
-#      - test_results/
 
 ### Tests ###
 tests:
@@ -100,6 +76,7 @@ tests:
   before_script:
     - chmod +x ./CI/update_badge.sh
     - ./CI/update_badge.sh > /dev/null
+    - source /opt/venv/bin/activate
   script:
     - pip install -r requirements.txt
     - chmod +x ./CI/run_pytest.sh
@@ -125,6 +102,7 @@ coverage:
   before_script:
     - chmod +x ./CI/update_badge.sh
     - ./CI/update_badge.sh > /dev/null
+    - source /opt/venv/bin/activate
   script:
     - pip install -r requirements.txt
     - chmod +x ./CI/run_pytest_coverage.sh
@@ -148,6 +126,7 @@ sphinx docs:
   before_script:
     - chmod +x ./CI/update_badge.sh
     - ./CI/update_badge.sh > /dev/null
+    - source /opt/venv/bin/activate
   script:
     - pip install -r requirements.txt
     - pip install -r docs/requirements_docs.txt

CHANGELOG.md

 # Changelog
 All notable changes to this project will be documented in this file.
 
+## v2.1.0 -  2022-06-07  - new evaluation metrics and improved training
+
+### general:
+* new evaluation metrics, IOA and MNMB
+* advanced train options for early stopping
+* reduced execution time by refactoring
+
+### new features:
+* uncertainty estimation of MSE is now applied for each season separately (#374)
+* added different configurations of early stopping to use either last trained or best epoch (#378)
+* train monitoring plots now add a star for best epoch when using early stopping (#367)
+* new evaluation metric index of agreement, IOA (#376)
+* new evaluation metric modified normalised mean bias, MNMB (#380)
+* new plot available that shows temporal evolution of MSE for each station (#381)
+
+### technical:
+* reduced loading of forecast path from data store (#328)
+* bug fix for not catched error during transformation (#385)
+* bug fix for data handler with climate and fir filter leading to calculate transformation always with fir filter (#387)
+* improved duration for latex report creation at end of preprocessing (#388)
+* enhanced speed for make prediction in postprocessing (#389)
+* fix to always create version badge from version and not from tag name (#382)
+
 ## v2.0.0 -  2022-04-08  - tf2 usage, new model classes, and improved uncertainty estimate
 
 ### general:

CI/Dockerfile

+# ---- base node ----
+FROM opensuse/leap:latest AS base
+MAINTAINER Lukas Leufen <l.leufen@fz-juelich.de>
+
+# install git
+RUN zypper --non-interactive install git
+
+# install python3
+RUN zypper --non-interactive install python39 python39-devel
+
+# install pip
+RUN zypper --non-interactive install python39-pip
+
+# upgrade pip
+RUN pip3.9 install --upgrade pip
+
+# install curl
+RUN zypper --non-interactive install curl
+
+# install make
+RUN zypper --non-interactive install make
+
+# install gcc
+RUN zypper --non-interactive install gcc-c++
+
+# create and activate venv
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3.9 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+# RUN source venv/bin/activate
+
+# ---- test node ----
+FROM base AS test
+
+# install pytest
+RUN pip install pytest pytest-html pytest-lazy-fixture
+
+# ---- coverage node ----
+FROM test AS coverage
+
+# install pytest coverage
+RUN pip install pytest-cov
+
+# ---- docs node ----
+FROM base AS docs
+
+# install sphinx
+RUN pip install sphinx
+
+# ---- MLAir ready to use ----
+FROM base AS mlair
+
+# install geo packages
+RUN zypper --no-gpg-checks addrepo https://download.opensuse.org/repositories/Application:Geo/15.4/Application:Geo.repo
+RUN zypper --no-gpg-checks refresh
+RUN zypper --no-gpg-checks --non-interactive install proj=8.2.1
+RUN zypper --no-gpg-checks --non-interactive install geos=3.10.3
+RUN zypper --no-gpg-checks --non-interactive install geos-devel=3.9.1
+RUN zypper --no-gpg-checks --non-interactive install libproj22=8.2.1
+RUN zypper --no-gpg-checks --non-interactive install binutils libproj-devel gdal-devel
+
+# install requirements
+ADD requirements.txt .
+RUN pip install -r requirements.txt
+

CI/create_version_badge.sh

 #!/bin/bash
 
-VERSION="$(git describe master)"
+VERSION="$(git describe --match v[0-9]*.[0-9]*.[0-9]* --abbrev=0)"
 COLOR="blue"
 BADGE_NAME="version"
 

CI/run_pytest.sh

 #!/bin/bash
 
 # run pytest for all run_modules
-python3.6 -m pytest --html=report.html --self-contained-html test/ | tee test_results.out
+python -m pytest --html=report.html --self-contained-html test/ | tee test_results.out
 
 IS_FAILED=$?
 

CI/run_pytest_coverage.sh

 #!/usr/bin/env bash
 
 # run coverage twice, 1) for html deploy 2) for success evaluation
-python3.6 -m pytest --cov=mlair --cov-report term  --cov-report html test/ | tee coverage_results.out
+python -m pytest --cov=mlair --cov-report term  --cov-report html test/ | tee coverage_results.out
 
 IS_FAILED=$?
 

HPC_setup/create_runscripts_HPC.sh

@@ -85,7 +85,7 @@ source venv_${hpcsys}/bin/activate
 
 timestamp=\`date +"%Y-%m-%d_%H%M-%S"\`
 
-export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.8/site-packages:\${PYTHONPATH}
+export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.9/site-packages:\${PYTHONPATH}
 
 srun --cpu-bind=none python run.py --experiment_date=\$timestamp
 EOT
@@ -111,7 +111,7 @@ source venv_${hpcsys}/bin/activate
 
 timestamp=\`date +"%Y-%m-%d_%H%M-%S"\`
 
-export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.8/site-packages:\${PYTHONPATH}
+export PYTHONPATH=\${PWD}/venv_${hpcsys}/lib/python3.9/site-packages:\${PYTHONPATH}
 
 srun --cpu-bind=none python run_HPC.py --experiment_date=\$timestamp
 EOT

HPC_setup/mlt_modules_hdfml.sh

@@ -8,13 +8,12 @@
 module --force purge
 module use $OTHERSTAGES
 
-ml Stages/2020
-ml GCCcore/.10.3.0
+ml Stages/2022
+ml GCCcore/.11.2.0
 
-ml Jupyter/2021.3.1-Python-3.8.5
-ml Python/3.8.5
-ml TensorFlow/2.5.0-Python-3.8.5
-ml SciPy-Stack/2021-Python-3.8.5
-ml dask/2.22.0-Python-3.8.5
-ml GEOS/3.8.1-Python-3.8.5
-ml Graphviz/2.44.1
\ No newline at end of file
+ml Python/3.9.6
+ml TensorFlow/2.6.0-CUDA-11.5
+ml dask/2021.9.1
+ml GEOS/3.9.1
+ml Cartopy/0.20.0
+ml Graphviz/2.49.3

HPC_setup/mlt_modules_juwels.sh

@@ -8,13 +8,12 @@
 module --force purge
 module use $OTHERSTAGES
 
-ml Stages/2020
-ml GCCcore/.10.3.0
+ml Stages/2022
+ml GCCcore/.11.2.0
 
-ml Jupyter/2021.3.1-Python-3.8.5
-ml Python/3.8.5
-ml TensorFlow/2.5.0-Python-3.8.5
-ml SciPy-Stack/2021-Python-3.8.5
-ml dask/2.22.0-Python-3.8.5
-ml GEOS/3.8.1-Python-3.8.5
-ml Graphviz/2.44.1
\ No newline at end of file
+ml Python/3.9.6
+ml TensorFlow/2.6.0-CUDA-11.5
+ml dask/2021.9.1
+ml GEOS/3.9.1
+ml Cartopy/0.20.0
+ml Graphviz/2.49.3

HPC_setup/requirements_HDFML_additionals.txt

-astropy==4.1
-bottleneck==1.3.2
-cached-property==1.5.2
-iniconfig==1.1.1
-ordered-set==4.0.2
-pyshp==2.1.3
-pytest-html==3.1.1
-pytest-lazy-fixture==0.6.3
-pytest-metadata==1.11.0
-pytest-sugar==0.9.4
-tabulate==0.8.8
+astropy==5.1
+pytz==2022.1
+python-dateutil==2.8.2
+requests==2.28.1
+werkzeug>=0.11.15
+wheel>=0.26
+six==1.15.0
+psutil==5.9.1
+pyparsing==3.0.9
+packaging==21.3
+timezonefinder==5.2.0
+patsy==0.5.2
+statsmodels==0.13.2
+seaborn==0.11.2
+xarray==0.16.2
+tabulate==0.8.10
 wget==3.2
---no-binary shapely Shapely==1.7.0
-
-#Cartopy==0.18.0
+pydot==1.4.2
+netcdf4==1.6.0
\ No newline at end of file

HPC_setup/requirements_JUWELS_additionals.txt

-astropy==4.1
-bottleneck==1.3.2
-cached-property==1.5.2
-iniconfig==1.1.1
-ordered-set==4.0.2
-pyshp==2.1.3
-pytest-html==3.1.1
-pytest-lazy-fixture==0.6.3
-pytest-metadata==1.11.0
-pytest-sugar==0.9.4
-tabulate==0.8.8
+astropy==5.1
+pytz==2022.1
+python-dateutil==2.8.2
+requests==2.28.1
+werkzeug>=0.11.15
+wheel>=0.26
+six==1.15.0
+psutil==5.9.1
+pyparsing==3.0.9
+packaging==21.3
+timezonefinder==5.2.0
+patsy==0.5.2
+statsmodels==0.13.2
+seaborn==0.11.2
+xarray==0.16.2
+tabulate==0.8.10
 wget==3.2
---no-binary shapely Shapely==1.7.0
-
-#Cartopy==0.18.0
+pydot==1.4.2
+netcdf4==1.6.0
\ No newline at end of file

HPC_setup/setup_venv_hdfml.sh

@@ -22,7 +22,7 @@ python3 -m venv ${cur}../venv_hdfml
 source ${cur}/../venv_hdfml/bin/activate
 
 # export path for side-packages 
-export PYTHONPATH=${cur}/../venv_hdfml/lib/python3.8/site-packages:${PYTHONPATH}
+export PYTHONPATH=${cur}/../venv_hdfml/lib/python3.9/site-packages:${PYTHONPATH}
 
 echo "##### START INSTALLING requirements_HDFML_additionals.txt #####"
 pip install -r ${cur}/requirements_HDFML_additionals.txt

HPC_setup/setup_venv_juwels.sh

@@ -22,7 +22,7 @@ python3 -m venv ${cur}/../venv_juwels
 source ${cur}/../venv_juwels/bin/activate
 
 # export path for side-packages 
-export PYTHONPATH=${cur}/../venv_juwels/lib/python3.8/site-packages:${PYTHONPATH}
+export PYTHONPATH=${cur}/../venv_juwels/lib/python3.9/site-packages:${PYTHONPATH}
 
 
 echo "##### START INSTALLING requirements_JUWELS_additionals.txt #####"

README.md

@@ -34,7 +34,7 @@ HPC systems, see [here](#special-instructions-for-installation-on-jülich-hpc-sy
 * Installation of **MLAir**:
     * Either clone MLAir from the [gitlab repository](https://gitlab.jsc.fz-juelich.de/esde/machine-learning/mlair.git) 
       and use it without installation (beside the requirements) 
-    * or download the distribution file ([current version](https://gitlab.jsc.fz-juelich.de/esde/machine-learning/mlair/-/blob/master/dist/mlair-2.0.0-py3-none-any.whl)) 
+    * or download the distribution file ([current version](https://gitlab.jsc.fz-juelich.de/esde/machine-learning/mlair/-/blob/master/dist/mlair-2.1.0-py3-none-any.whl)) 
       and install it via `pip install <dist_file>.whl`. In this case, you can simply import MLAir in any python script 
       inside your virtual environment using `import mlair`.
 

docs/_source/installation.rst

@@ -27,7 +27,7 @@ Installation of MLAir
 * Install all requirements from `requirements.txt <https://gitlab.jsc.fz-juelich.de/esde/machine-learning/mlair/-/blob/master/requirements.txt>`_
   preferably in a virtual environment
 * Either clone MLAir from the `gitlab repository <https://gitlab.jsc.fz-juelich.de/esde/machine-learning/mlair.git>`_
-* or download the distribution file (`current version <https://gitlab.jsc.fz-juelich.de/esde/machine-learning/mlair/-/blob/master/dist/mlair-2.0.0-py3-none-any.whl>`_)
+* or download the distribution file (`current version <https://gitlab.jsc.fz-juelich.de/esde/machine-learning/mlair/-/blob/master/dist/mlair-2.1.0-py3-none-any.whl>`_)
   and install it via :py:`pip install <dist_file>.whl`. In this case, you can simply
   import MLAir in any python script inside your virtual environment using :py:`import mlair`.
 

docs/requirements_docs.txt

@@ -2,8 +2,8 @@ sphinx==3.0.3
 sphinx-autoapi==1.8.4
 sphinx-autodoc-typehints==1.12.0
 sphinx-rtd-theme==0.4.3
-#recommonmark==0.6.0
 m2r2==0.3.1
 docutils<0.18
 mistune==0.8.4
 setuptools>=59.5.0
+Jinja2<3.1
\ No newline at end of file

mlair/__init__.py

 __version_info__ = {
     'major': 2,
-    'minor': 0,
+    'minor': 1,
     'micro': 0,
 }
 

mlair/configuration/defaults.py

@@ -9,7 +9,6 @@ DEFAULT_STATIONS = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087']
 DEFAULT_VAR_ALL_DICT = {'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum', 'u': 'average_values',
                         'v': 'average_values', 'no': 'dma8eu', 'no2': 'dma8eu', 'cloudcover': 'average_values',
                         'pblheight': 'maximum'}
-DEFAULT_NETWORK = "AIRBASE"
 DEFAULT_STATION_TYPE = "background"
 DEFAULT_VARIABLES = DEFAULT_VAR_ALL_DICT.keys()
 DEFAULT_START = "1997-01-01"

mlair/configuration/era5_settings.py

+"""Settings to access not public era5 data."""
+
+from typing import Tuple
+
+
+def era5_settings(sampling="daily") -> Tuple[str, str]:
+    """
+    Check for sampling as only hourly resolution is supported by era5 and return path on HPC systems.
+
+    :param sampling: temporal resolution to load data for, only hourly supported (default "daily")
+
+    :return: HPC path
+    """
+    if sampling == "hourly":  # pragma: no branch
+        ERA5_DATA_PATH = "."
+        FILE_NAMES = "*.nc"
+    else:
+        raise NameError(f"Given sampling {sampling} is not supported, only hourly sampling can be used.")
+    return ERA5_DATA_PATH, FILE_NAMES