diff --git a/.gitignore b/.gitignore
index f7793d5f492cced655aeb62a8c29af48ac3e452e..222931f853f9ddf2e25dbfb6c26f1051c456bef4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,9 +45,10 @@ Thumbs.db
/data/
/plots/
-# tmp folder #
-##############
+# tmp and logging folder #
+##########################
/tmp/
+/logging/
# test related data #
#####################
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9f1ba73f5a3b3fada8624dfd89fa6f12488a877b..f3ec1ab98cf8e46b97e2d803518ed57c6cfd4622 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -23,13 +23,71 @@ version:
paths:
- badges/
+### Tests (from scratch) ###
+tests (from scratch):
+ tags:
+ - base
+ - zam347
+ stage: test
+ only:
+ - master
+ - /^release.*$/
+ - develop
+ variables:
+ FAILURE_THRESHOLD: 100
+ TEST_TYPE: "scratch"
+ before_script:
+ - chmod +x ./CI/update_badge.sh
+ - ./CI/update_badge.sh > /dev/null
+ script:
+ - zypper --non-interactive install binutils libproj-devel gdal-devel
+ - zypper --non-interactive install proj geos-devel
+ - pip install -r requirements.txt
+ - chmod +x ./CI/run_pytest.sh
+ - ./CI/run_pytest.sh
+ after_script:
+ - ./CI/update_badge.sh > /dev/null
+ artifacts:
+ name: pages
+ when: always
+ paths:
+ - badges/
+ - test_results/
+
+### Tests (on GPU) ###
+tests (on GPU):
+ tags:
+ - gpu
+ - zam347
+ stage: test
+ only:
+ - master
+ - /^release.*$/
+ - develop
+ variables:
+ FAILURE_THRESHOLD: 100
+ TEST_TYPE: "gpu"
+ before_script:
+ - chmod +x ./CI/update_badge.sh
+ - ./CI/update_badge.sh > /dev/null
+ script:
+ - pip install -r requirements.txt
+ - chmod +x ./CI/run_pytest.sh
+ - ./CI/run_pytest.sh
+ after_script:
+ - ./CI/update_badge.sh > /dev/null
+ artifacts:
+ name: pages
+ when: always
+ paths:
+ - badges/
+ - test_results/
+
### Tests ###
tests:
tags:
- - leap
+ - machinelearningtools
- zam347
- - base
- - django
stage: test
variables:
FAILURE_THRESHOLD: 100
@@ -51,10 +109,8 @@ tests:
coverage:
tags:
- - leap
+ - machinelearningtools
- zam347
- - base
- - django
stage: test
variables:
FAILURE_THRESHOLD: 50
@@ -79,7 +135,6 @@ coverage:
pages:
stage: pages
tags:
- - leap
- zam347
- base
script:
diff --git a/CI/run_pytest.sh b/CI/run_pytest.sh
index a7d883dcc95e0c16541af00ed0891e2d31dee82c..baa7ef8e892fc2d9efdd30094917ca492017de3d 100644
--- a/CI/run_pytest.sh
+++ b/CI/run_pytest.sh
@@ -1,24 +1,30 @@
#!/bin/bash
# run pytest for all run_modules
-python3 -m pytest --html=report.html --self-contained-html test/ | tee test_results.out
+python3.6 -m pytest --html=report.html --self-contained-html test/ | tee test_results.out
IS_FAILED=$?
+if [ -z ${TEST_TYPE+x} ]; then
+ TEST_TYPE=""; else
+ TEST_TYPE="_"${TEST_TYPE};
+fi
+
# move html test report
-mkdir test_results/
+TEST_RESULTS_DIR="test_results${TEST_TYPE}/"
+mkdir ${TEST_RESULTS_DIR}
BRANCH_NAME=$( echo -e "${CI_COMMIT_REF_NAME////_}")
-mkdir test_results/${BRANCH_NAME}
-mkdir test_results/recent
-cp report.html test_results/${BRANCH_NAME}/.
-cp report.html test_results/recent/.
+mkdir "${TEST_RESULTS_DIR}/${BRANCH_NAME}"
+mkdir "${TEST_RESULTS_DIR}/recent"
+cp report.html "${TEST_RESULTS_DIR}/${BRANCH_NAME}/."
+cp report.html "${TEST_RESULTS_DIR}/recent/."
if [[ "${CI_COMMIT_REF_NAME}" = "master" ]]; then
- cp -r report.html test_results/.
+ cp -r report.html ${TEST_RESULTS_DIR}/.
fi
# exit 0 if no tests implemented
RUN_NO_TESTS="$(grep -c 'no tests ran' test_results.out)"
-if [[ ${RUN_NO_TESTS} > 0 ]]; then
+if [[ ${RUN_NO_TESTS} -gt 0 ]]; then
echo "no test available"
echo "incomplete" > status.txt
echo "no tests avail" > incomplete.txt
@@ -27,20 +33,19 @@ fi
# extract if tests passed or not
TEST_FAILED="$(grep -oP '(\d+\s{1}failed)' test_results.out)"
-TEST_FAILED="$(echo ${TEST_FAILED} | (grep -oP '\d*'))"
+TEST_FAILED="$(echo "${TEST_FAILED}" | (grep -oP '\d*'))"
TEST_PASSED="$(grep -oP '\d+\s{1}passed' test_results.out)"
-TEST_PASSED="$(echo ${TEST_PASSED} | (grep -oP '\d*'))"
+TEST_PASSED="$(echo "${TEST_PASSED}" | (grep -oP '\d*'))"
if [[ -z "$TEST_FAILED" ]]; then
TEST_FAILED=0
fi
-let "TEST_PASSED=${TEST_PASSED}-${TEST_FAILED}"
-
+(( TEST_PASSED=TEST_PASSED-TEST_FAILED ))
# calculate metrics
-let "SUM=${TEST_FAILED}+${TEST_PASSED}"
-let "TEST_PASSED_RATIO=${TEST_PASSED}*100/${SUM}"
+(( SUM=TEST_FAILED+TEST_PASSED ))
+(( TEST_PASSED_RATIO=TEST_PASSED*100/SUM ))
# report
-if [[ ${IS_FAILED} == 0 ]]; then
+if [[ ${IS_FAILED} -eq 0 ]]; then
if [[ ${TEST_PASSED_RATIO} -lt 100 ]]; then
echo "only ${TEST_PASSED_RATIO}% passed"
echo "incomplete" > status.txt
diff --git a/CI/run_pytest_coverage.sh b/CI/run_pytest_coverage.sh
index 2157192d49a15baa048968b799aa264941152c1b..45916427f1521843923fb94e49dc661241dc0369 100644
--- a/CI/run_pytest_coverage.sh
+++ b/CI/run_pytest_coverage.sh
@@ -1,17 +1,16 @@
#!/usr/bin/env bash
# run coverage twice, 1) for html deploy 2) for success evaluation
-python3 -m pytest --cov=src --cov-report html test/
-python3 -m pytest --cov=src --cov-report term test/ | tee coverage_results.out
+python3.6 -m pytest --cov=src --cov-report term --cov-report html test/ | tee coverage_results.out
IS_FAILED=$?
# move html coverage report
mkdir coverage/
BRANCH_NAME=$( echo -e "${CI_COMMIT_REF_NAME////_}")
-mkdir coverage/${BRANCH_NAME}
-mkdir coverage/recent
-cp -r htmlcov/* coverage/${BRANCH_NAME}/.
+mkdir "coverage/${BRANCH_NAME}"
+mkdir "coverage/recent"
+cp -r htmlcov/* "coverage/${BRANCH_NAME}/."
cp -r htmlcov/* coverage/recent/.
if [[ "${CI_COMMIT_REF_NAME}" = "master" ]]; then
cp -r htmlcov/* coverage/.
@@ -19,10 +18,10 @@ fi
# extract coverage information
COVERAGE_RATIO="$(grep -oP '\d+\%' coverage_results.out | tail -1)"
-COVERAGE_RATIO="$(echo ${COVERAGE_RATIO} | (grep -oP '\d*'))"
+COVERAGE_RATIO="$(echo "${COVERAGE_RATIO}" | (grep -oP '\d*'))"
# report
-if [[ ${IS_FAILED} == 0 ]]; then
+if [[ ${IS_FAILED} -eq 0 ]]; then
if [[ ${COVERAGE_RATIO} -lt ${COVERAGE_PASS_THRESHOLD} ]]; then
echo "only ${COVERAGE_RATIO}% covered"
echo "incomplete" > status.txt
diff --git a/German_background_stations.json b/German_background_stations.json
new file mode 100755
index 0000000000000000000000000000000000000000..2997eefbaa9a72f4e94b940b6d0ebb7f6a34370d
--- /dev/null
+++ b/German_background_stations.json
@@ -0,0 +1 @@
+["DENW094", "DEBW029", "DENI052", "DENI063", "DEBY109", "DEUB022", "DESN001", "DEUB013", "DETH016", "DEBY002", "DEBY005", "DEBY099", "DEUB038", "DEBE051", "DEBE056", "DEBE062", "DEBE032", "DEBE034", "DEBE010", "DEHE046", "DEST031", "DEBY122", "DERP022", "DEBY079", "DEBW102", "DEBW076", "DEBW045", "DESH016", "DESN004", "DEHE032", "DEBB050", "DEBW042", "DEBW046", "DENW067", "DESL019", "DEST014", "DENW062", "DEHE033", "DENW081", "DESH008", "DEBB055", "DENI011", "DEHB001", "DEHB004", "DEHB002", "DEHB003", "DEHB005", "DEST039", "DEUB003", "DEBW072", "DEST002", "DEBB001", "DEHE039", "DEBW035", "DESN005", "DEBW047", "DENW004", "DESN011", "DESN076", "DEBB064", "DEBB006", "DEHE001", "DESN012", "DEST030", "DESL003", "DEST104", "DENW050", "DENW008", "DETH026", "DESN085", "DESN014", "DESN092", "DENW071", "DEBW004", "DENI028", "DETH013", "DENI059", "DEBB007", "DEBW049", "DENI043", "DETH020", "DEBY017", "DEBY113", "DENW247", "DENW028", "DEBW025", "DEUB039", "DEBB009", "DEHE027", "DEBB042", "DEHE008", "DESN017", "DEBW084", "DEBW037", "DEHE058", "DEHE028", "DEBW112", "DEBY081", "DEBY082", "DEST032", "DETH009", "DEHE010", "DESN019", "DEHE023", "DETH036", "DETH040", "DEMV017", "DEBW028", "DENI042", "DEMV004", "DEMV019", "DEST044", "DEST050", "DEST072", "DEST022", "DEHH049", "DEHH047", "DEHH033", "DEHH050", "DEHH008", "DEHH021", "DENI054", "DEST070", "DEBB053", "DENW029", "DEBW050", "DEUB034", "DENW018", "DEST052", "DEBY020", "DENW063", "DESN050", "DETH061", "DERP014", "DETH024", "DEBW094", "DENI031", "DETH041", "DERP019", "DEBW081", "DEHE013", "DEBW021", "DEHE060", "DEBY031", "DESH021", "DESH033", "DEHE052", "DEBY004", "DESN024", "DEBW052", "DENW042", "DEBY032", "DENW053", "DENW059", "DEBB082", "DEBB031", "DEHE025", "DEBW053", "DEHE048", "DENW051", "DEBY034", "DEUB035", "DEUB032", "DESN028", "DESN059", "DEMV024", "DENW079", "DEHE044", "DEHE042", "DEBB043", "DEBB036", "DEBW024", "DERP001", "DEMV012", "DESH005", "DESH023", "DEUB031", "DENI062", "DENW006", "DEBB065", "DEST077", "DEST005", "DERP007", "DEBW006", "DEBW007", "DEHE030", "DENW015", "DEBY013", "DETH025", "DEUB033", "DEST025", "DEHE045", "DESN057", "DENW036", "DEBW044", "DEUB036", "DENW096", "DETH095", "DENW038", "DEBY089", "DEBY039", "DENW095", "DEBY047", "DEBB067", "DEBB040", "DEST078", "DENW065", "DENW066", "DEBY052", "DEUB030", "DETH027", "DEBB048", "DENW047", "DEBY049", "DERP021", "DEHE034", "DESN079", "DESL008", "DETH018", "DEBW103", "DEHE017", "DEBW111", "DENI016", "DENI038", "DENI058", "DENI029", "DEBY118", "DEBW032", "DEBW110", "DERP017", "DESN036", "DEBW026", "DETH042", "DEBB075", "DEBB052", "DEBB021", "DEBB038", "DESN051", "DEUB041", "DEBW020", "DEBW113", "DENW078", "DEHE018", "DEBW065", "DEBY062", "DEBW027", "DEBW041", "DEHE043", "DEMV007", "DEMV021", "DEBW054", "DETH005", "DESL012", "DESL011", "DEST069", "DEST071", "DEUB004", "DESH006", "DEUB029", "DEUB040", "DESN074", "DEBW031", "DENW013", "DENW179", "DEBW056", "DEBW087", "DEST061", "DEMV001", "DEBB024", "DEBW057", "DENW064", "DENW068", "DENW080", "DENI019", "DENI077", "DEHE026", "DEBB066", "DEBB083", "DEST063", "DEBW013", "DETH086", "DESL018", "DETH096", "DEBW059", "DEBY072", "DEBY088", "DEBW060", "DEBW107", "DEBW036", "DEUB026", "DEBW019", "DENW010", "DEST098", "DEHE019", "DEBW039", "DESL017", "DEBW034", "DEUB005", "DEBB051", "DEHE051", "DEBW023", "DEBY092", "DEBW008", "DEBW030", "DENI060", "DEST011", "DENW030", "DENI041", "DERP015", "DEUB001", "DERP016", "DERP028", "DERP013", "DEHE022", "DEUB021", "DEBW010", "DEST066", "DEBB063", "DEBB028", "DEHE024", "DENI020", "DENI051", "DERP025", "DEBY077", "DEMV018", "DEST089", "DEST028", "DETH060", "DEHE050", "DEUB028", "DESN045", "DEUB042"]
diff --git a/requirements.txt b/requirements.txt
index 607563fd9042d68805c2be238ba982d62ba28974..b46f44416cf6560ecc0b62f8d22dd7d547a036c6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -43,17 +43,20 @@ pytest-cov==2.8.1
pytest-html==2.0.1
pytest-lazy-fixture==0.6.3
pytest-metadata==1.8.0
+pytest-sugar
python-dateutil==2.8.1
pytz==2019.3
PyYAML==5.3
requests==2.23.0
scipy==1.4.1
seaborn==0.10.0
-Shapely==1.7.0
+--no-binary shapely Shapely==1.7.0
six==1.11.0
statsmodels==0.11.1
-tensorboard==1.12.2
-tensorflow==1.12.0
+tabulate
+tensorboard==1.13.1
+tensorflow-estimator==1.13.0
+tensorflow==1.13.1
termcolor==1.1.0
toolz==0.10.0
urllib3==1.25.8
diff --git a/requirements_gpu.txt b/requirements_gpu.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6ce4df8fe164408024e21db5ea94a692fb5dbf26
--- /dev/null
+++ b/requirements_gpu.txt
@@ -0,0 +1,66 @@
+absl-py==0.9.0
+astor==0.8.1
+atomicwrites==1.3.0
+attrs==19.3.0
+Cartopy==0.17.0
+certifi==2019.11.28
+chardet==3.0.4
+cloudpickle==1.3.0
+coverage==5.0.3
+cycler==0.10.0
+Cython==0.29.15
+dask==2.11.0
+fsspec==0.6.2
+gast==0.3.3
+grpcio==1.27.2
+h5py==2.10.0
+idna==2.8
+importlib-metadata==1.5.0
+Keras==2.2.4
+Keras-Applications==1.0.8
+Keras-Preprocessing==1.1.0
+kiwisolver==1.1.0
+locket==0.2.0
+Markdown==3.2.1
+matplotlib==3.2.0
+mock==4.0.1
+more-itertools==8.2.0
+numpy==1.18.1
+packaging==20.3
+pandas==1.0.1
+partd==1.1.0
+patsy==0.5.1
+Pillow==7.0.0
+pluggy==0.13.1
+protobuf==3.11.3
+py==1.8.1
+pydot==1.4.1
+pyparsing==2.4.6
+pyproj==2.5.0
+pyshp==2.1.0
+pytest==5.3.5
+pytest-cov==2.8.1
+pytest-html==2.0.1
+pytest-lazy-fixture==0.6.3
+pytest-metadata==1.8.0
+pytest-sugar
+python-dateutil==2.8.1
+pytz==2019.3
+PyYAML==5.3
+requests==2.23.0
+scipy==1.4.1
+seaborn==0.10.0
+--no-binary shapely Shapely==1.7.0
+six==1.11.0
+statsmodels==0.11.1
+tabulate
+tensorboard==1.13.1
+tensorflow-estimator==1.13.0
+tensorflow-gpu==1.13.1
+termcolor==1.1.0
+toolz==0.10.0
+urllib3==1.25.8
+wcwidth==0.1.8
+Werkzeug==1.0.0
+xarray==0.15.0
+zipp==3.1.0
diff --git a/run.py b/run.py
index 556cd0b59ed023178fa7e6df1b5b03b9f6c5f157..9809712876dc886007b042a52d7b46c027800faf 100644
--- a/run.py
+++ b/run.py
@@ -3,7 +3,6 @@ __date__ = '2019-11-14'
import argparse
-import logging
from src.run_modules.experiment_setup import ExperimentSetup
from src.run_modules.model_setup import ModelSetup
@@ -17,7 +16,8 @@ def main(parser_args):
with RunEnvironment():
ExperimentSetup(parser_args, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBW001'],
- station_type='background', trainable=False, create_new_model=False)
+ station_type='background', trainable=False, create_new_model=False, window_history_size=6,
+ create_new_bootstraps=True)
PreProcessing()
ModelSetup()
@@ -29,10 +29,6 @@ def main(parser_args):
if __name__ == "__main__":
- formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
- logging.basicConfig(format=formatter, level=logging.INFO)
- # logging.basicConfig(format=formatter, level=logging.DEBUG)
-
parser = argparse.ArgumentParser()
parser.add_argument('--experiment_date', metavar='--exp_date', type=str, default=None,
help="set experiment date as string")
diff --git a/run_hourly.py b/run_hourly.py
index af531aedbd275b133a087777334dba0ae24bd9c8..3c3135c46df9875633499bd17b237a23cdf6be55 100644
--- a/run_hourly.py
+++ b/run_hourly.py
@@ -29,10 +29,6 @@ def main(parser_args):
if __name__ == "__main__":
- formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
- logging.basicConfig(format=formatter, level=logging.INFO)
- # logging.basicConfig(format=formatter, level=logging.DEBUG)
-
parser = argparse.ArgumentParser()
parser.add_argument('--experiment_date', metavar='--exp_date', type=str, default=None,
help="set experiment date as string")
diff --git a/run_zam347.py b/run_zam347.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e140f48188a6df7207e04d048f38d9701c69d4b
--- /dev/null
+++ b/run_zam347.py
@@ -0,0 +1,55 @@
+__author__ = "Lukas Leufen"
+__date__ = '2019-11-14'
+
+
+import argparse
+import json
+import logging
+
+from src.run_modules.experiment_setup import ExperimentSetup
+from src.run_modules.model_setup import ModelSetup
+from src.run_modules.post_processing import PostProcessing
+from src.run_modules.pre_processing import PreProcessing
+from src.run_modules.run_environment import RunEnvironment
+from src.run_modules.training import Training
+
+
+def load_stations():
+
+ try:
+ filename = 'German_background_stations.json'
+ with open(filename, 'r') as jfile:
+ stations = json.load(jfile)
+ logging.info(f"load station IDs from file: {filename} ({len(stations)} stations)")
+ # stations.remove('DEUB042')
+ except FileNotFoundError:
+ stations = ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBW001']
+ # stations = ['DEBB050', 'DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBW001']
+ logging.info(f"Use stations from list: {stations}")
+
+ return stations
+
+
+def main(parser_args):
+
+ with RunEnvironment():
+
+ ExperimentSetup(parser_args, stations=load_stations(), station_type='background', trainable=False,
+ create_new_model=True)
+ PreProcessing()
+
+ ModelSetup()
+
+ Training()
+
+ PostProcessing()
+
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--experiment_date', metavar='--exp_date', type=str, default=None,
+ help="set experiment date as string")
+ args = parser.parse_args(["--experiment_date", "testrun"])
+
+ main(args)
diff --git a/src/data_handling/bootstraps.py b/src/data_handling/bootstraps.py
index 3e69950267f9c95ccc636e560a21731ade388432..46fa7c2be39d3dadb1922a1b710065aa42d9e2d2 100644
--- a/src/data_handling/bootstraps.py
+++ b/src/data_handling/bootstraps.py
@@ -2,152 +2,108 @@ __author__ = 'Felix Kleinert, Lukas Leufen'
__date__ = '2020-02-07'
-from src.run_modules.run_environment import RunEnvironment
from src.data_handling.data_generator import DataGenerator
import numpy as np
import logging
+import keras
import dask.array as da
import xarray as xr
import os
import re
from src import helpers
+from typing import List, Union, Pattern, Tuple
-class BootStrapGenerator:
+class BootStrapGenerator(keras.utils.Sequence):
+ """
+ generator for bootstraps as keras sequence inheritance. Initialise with number of boots, the original history, the
+ shuffled data, all used variables and the current shuffled variable. While iterating over this generator, it returns
+ the bootstrapped history for given boot index (this is the iterator index) in the same format like the original
+ history ready to use. Note, that in some cases some samples can contain nan values (in these cases the entire data
+ row is null, not only single entries).
+ """
+ def __init__(self, number_of_boots: int, history: xr.DataArray, shuffled: xr.DataArray, variables: List[str],
+ shuffled_variable: str):
+ self.number_of_boots = number_of_boots
+ self.variables = variables
+ self.history_orig = history
+ self.history = history.sel(variables=helpers.list_pop(self.variables, shuffled_variable))
+ self.shuffled = shuffled.sel(variables=shuffled_variable)
- def __init__(self, orig_generator, boots, chunksize, bootstrap_path):
- self.orig_generator: DataGenerator = orig_generator
- self.stations = self.orig_generator.stations
- self.variables = self.orig_generator.variables
- self.boots = boots
- self.chunksize = chunksize
- self.bootstrap_path = bootstrap_path
- self._iterator = 0
- self.bootstrap_meta = []
-
- def __len__(self):
- """
- display the number of stations
- """
- return len(self.orig_generator)*self.boots*len(self.variables)
-
- def get_labels(self, key):
- _, label = self.orig_generator[key]
- for _ in range(self.boots):
- yield label
-
- def get_generator(self):
- """
- This is the implementation of the __next__ method of the iterator protocol. Get the data generator, and return
- the history and label data of this generator.
- :return:
- """
- while True:
- for i, data in enumerate(self.orig_generator):
- station = self.orig_generator.get_station_key(i)
- logging.info(f"station: {station}")
- hist, label = data
- len_of_label = len(label)
- shuffled_data = self.load_boot_data(station)
- for var in self.variables:
- logging.info(f" var: {var}")
- for boot in range(self.boots):
- logging.debug(f"boot: {boot}")
- boot_hist = hist.sel(variables=helpers.list_pop(self.variables, var))
- shuffled_var = shuffled_data.sel(variables=var, boots=boot).expand_dims("variables").drop("boots").transpose("datetime", "window", "Stations", "variables")
- boot_hist = boot_hist.combine_first(shuffled_var)
- boot_hist = boot_hist.sortby("variables")
- self.bootstrap_meta.extend([[var, station]]*len_of_label)
- yield boot_hist, label
- return
-
- def get_orig_prediction(self, path, file_name, prediction_name="CNN"):
- file = os.path.join(path, file_name)
- data = xr.open_dataarray(file)
- for _ in range(self.boots):
- yield data.sel(type=prediction_name).squeeze()
-
- def load_boot_data(self, station):
- files = os.listdir(self.bootstrap_path)
- regex = re.compile(rf"{station}_\w*\.nc")
- file_name = os.path.join(self.bootstrap_path, list(filter(regex.search, files))[0])
- shuffled_data = xr.open_dataarray(file_name, chunks=100)
- return shuffled_data
+ def __len__(self) -> int:
+ return self.number_of_boots
+ def __getitem__(self, index: int) -> xr.DataArray:
+ """
+ return bootstrapped history for given bootstrap index in same index structure like the original history object
+ :param index: boot index e [0, nboots-1]
+ :return: bootstrapped history ready to use
+ """
+ logging.debug(f"boot: {index}")
+ boot_hist = self.history.copy()
+ boot_hist = boot_hist.combine_first(self.__get_shuffled(index))
+ return boot_hist.reindex_like(self.history_orig)
-class BootStraps(RunEnvironment):
+ def __get_shuffled(self, index: int) -> xr.DataArray:
+ """
+ returns shuffled data for given boot index from shuffled attribute
+ :param index: boot index e [0, nboots-1]
+ :return: shuffled data
+ """
+ shuffled_var = self.shuffled.sel(boots=index).expand_dims("variables").drop("boots")
+ return shuffled_var.transpose("datetime", "window", "Stations", "variables")
- def __init__(self, data, bootstrap_path, number_bootstraps=10):
- super().__init__()
- self.data: DataGenerator = data
- self.number_bootstraps = number_bootstraps
+class CreateShuffledData:
+ """
+ Verify and create shuffled data for all data contained in given data generator class. Starts automatically on
+ initialisation, no further calls are required. Check and new creations are all performed inside bootstrap_path.
+ """
+ def __init__(self, data: DataGenerator, number_of_bootstraps: int, bootstrap_path: str):
+ self.data = data
+ self.number_of_bootstraps = number_of_bootstraps
self.bootstrap_path = bootstrap_path
- self.chunks = self.get_chunk_size()
self.create_shuffled_data()
- self._boot_strap_generator = BootStrapGenerator(self.data, self.number_bootstraps, self.chunks, self.bootstrap_path)
-
- def get_boot_strap_meta(self):
- return self._boot_strap_generator.bootstrap_meta
-
- def boot_strap_generator(self):
- return self._boot_strap_generator.get_generator()
-
- def get_boot_strap_generator_length(self):
- return self._boot_strap_generator.__len__()
-
- def get_labels(self, key):
- labels_list = []
- chunks = None
- for labels in self._boot_strap_generator.get_labels(key):
- if len(labels_list) == 0:
- chunks = (100, labels.data.shape[1])
- labels_list.append(da.from_array(labels.data, chunks=chunks))
- labels_out = da.concatenate(labels_list, axis=0)
- return labels_out.compute()
-
- def get_orig_prediction(self, path, name):
- labels_list = []
- chunks = None
- for labels in self._boot_strap_generator.get_orig_prediction(path, name):
- if len(labels_list) == 0:
- chunks = (100, labels.data.shape[1])
- labels_list.append(da.from_array(labels.data, chunks=chunks))
- labels_out = da.concatenate(labels_list, axis=0)
- labels_out = labels_out.compute()
- return labels_out[~np.isnan(labels_out).any(axis=1), :]
-
- def get_chunk_size(self):
- hist, _ = self.data[0]
- return (100, *hist.shape[1:], self.number_bootstraps)
-
- def create_shuffled_data(self):
+
+ def create_shuffled_data(self) -> None:
"""
Create shuffled data. Use original test data, add dimension 'boots' with length number of bootstraps and insert
randomly selected variables. If there is a suitable local file for requested window size and number of
bootstraps, no additional file will be created inside this function.
"""
- logging.info("create shuffled bootstrap data")
+ logging.info("create / check shuffled bootstrap data")
variables_str = '_'.join(sorted(self.data.variables))
window = self.data.window_history_size
for station in self.data.stations:
valid, nboot = self.valid_bootstrap_file(station, variables_str, window)
if not valid:
logging.info(f'create bootstap data for {station}')
- hist, _ = self.data[station]
- data = hist.copy()
- file_name = f"{station}_{variables_str}_hist{window}_nboots{nboot}_shuffled.nc"
- file_path = os.path.join(self.bootstrap_path, file_name)
- data = data.expand_dims({'boots': range(nboot)}, axis=-1)
+ hist = self.data.get_data_generator(station).get_transposed_history()
+ file_path = self._set_file_path(station, variables_str, window, nboot)
+ hist = hist.expand_dims({'boots': range(nboot)}, axis=-1)
shuffled_variable = []
- for i, var in enumerate(data.coords['variables']):
- single_variable = data.sel(variables=var).values
- shuffled_variable.append(self.shuffle_single_variable(single_variable, chunks=(100, *data.shape[1:3], data.shape[-1])))
- shuffled_variable_da = da.stack(shuffled_variable, axis=-2, ).rechunk("auto")
- shuffled_data = xr.DataArray(shuffled_variable_da, coords=data.coords, dims=data.dims)
+ chunks = (100, *hist.shape[1:3], hist.shape[-1])
+ for i, var in enumerate(hist.coords['variables']):
+ single_variable = hist.sel(variables=var).values
+ shuffled_variable.append(self.shuffle(single_variable, chunks=chunks))
+ shuffled_variable_da = da.stack(shuffled_variable, axis=-2).rechunk("auto")
+ shuffled_data = xr.DataArray(shuffled_variable_da, coords=hist.coords, dims=hist.dims)
shuffled_data.to_netcdf(file_path)
- def valid_bootstrap_file(self, station, variables, window):
+ def _set_file_path(self, station: str, variables: str, window: int, nboots: int) -> str:
+ """
+ Set file name following naming convention <station>_<var1>_<var2>_..._hist<window>_nboots<nboots>_shuffled.nc
+ and creates joined path using bootstrap_path attribute set on initialisation.
+ :param station: station name
+ :param variables: variables already preprocessed as single string with all variables seperated by underscore
+ :param window: window length
+ :param nboots: number of boots
+ :return: full file path
+ """
+ file_name = f"{station}_{variables}_hist{window}_nboots{nboots}_shuffled.nc"
+ return os.path.join(self.bootstrap_path, file_name)
+
+ def valid_bootstrap_file(self, station: str, variables: str, window: int) -> [bool, Union[None, int]]:
"""
Compare local bootstrap file with given settings for station, variables, window and number of bootstraps. If a
match was found, this method returns a tuple (True, None). In any other case, it returns (False, max_nboot),
@@ -155,32 +111,166 @@ class BootStraps(RunEnvironment):
length is ge than given window size form args and the number of boots is also ge than the given number of boots
from this class. Furthermore, this functions deletes local files, if the match the station pattern but don't fit
the window and bootstrap condition. This is performed, because it is assumed, that the corresponding file will
- be created with a longer or at least same window size and numbers of bootstraps.
- :param station:
- :param variables:
- :param window:
- :return:
+ be created with a longer or at the least same window size and numbers of bootstraps.
+ :param station: name of the station to validate
+ :param variables: all variables already merged in single string seperated by underscore
+ :param window: required window size
+ :return: tuple containing information if valid file was found first and second the number of boots that needs to
+ be used for the new boot creation (this is only relevant, if no valid file was found - otherwise the return
+ statement is anyway None).
"""
regex = re.compile(rf"{station}_{variables}_hist(\d+)_nboots(\d+)_shuffled")
- max_nboot = self.number_bootstraps
+ max_nboot = self.number_of_bootstraps
for file in os.listdir(self.bootstrap_path):
match = regex.match(file)
if match:
window_file = int(match.group(1))
nboot_file = int(match.group(2))
max_nboot = max([max_nboot, nboot_file])
- if (window_file >= window) and (nboot_file >= self.number_bootstraps):
+ if (window_file >= window) and (nboot_file >= self.number_of_bootstraps):
return True, None
else:
os.remove(os.path.join(self.bootstrap_path, file))
return False, max_nboot
@staticmethod
- def shuffle_single_variable(data: da.array, chunks) -> da.core.Array:
+ def shuffle(data: da.array, chunks: Tuple) -> da.core.Array:
+ """
+ Shuffle randomly from given data (draw elements with replacement)
+ :param data: data to shuffle
+ :param chunks: chunk size for dask
+ :return: shuffled data as dask core array (not computed yet)
+ """
size = data.shape
return da.random.choice(data.reshape(-1,), size=size, chunks=chunks)
+class BootStraps:
+ """
+ Main class to perform bootstrap operations. This class requires a DataGenerator object and a path, where to find and
+ store all data related to the bootstrap operation. In initialisation, this class will automatically call the class
+ CreateShuffleData to set up the shuffled data sets. How to use BootStraps:
+ * call .get_generator(<station>, <variable>) to get a generator for given station and variable combination that
+ iterates over all bootstrap realisations (as keras sequence)
+ * call .get_labels(<station>) to get the measured observations in the same format as bootstrap predictions
+ * call .get_bootstrap_predictions(<station>, <variable>) to get the bootstrapped predictions
+ * call .get_orig_prediction(<station>) to get the non-bootstrapped predictions (referred as original predictions)
+ """
+
+ def __init__(self, data: DataGenerator, bootstrap_path: str, number_of_bootstraps: int = 10):
+ self.data = data
+ self.number_of_bootstraps = number_of_bootstraps
+ self.bootstrap_path = bootstrap_path
+ CreateShuffledData(data, number_of_bootstraps, bootstrap_path)
+
+ @property
+ def stations(self) -> List[str]:
+ return self.data.stations
+
+ @property
+ def variables(self) -> List[str]:
+ return self.data.variables
+
+ @property
+ def window_history_size(self) -> int:
+ return self.data.window_history_size
+
+ def get_generator(self, station: str, variable: str) -> BootStrapGenerator:
+ """
+ Returns the actual generator to use for the bootstrap evaluation. The generator requires information on station
+ and bootstrapped variable. There is only a loop on the bootstrap realisation and not on stations or variables.
+ :param station: name of the station
+ :param variable: name of the variable to bootstrap
+ :return: BootStrapGenerator class ready to use.
+ """
+ hist, _ = self.data[station]
+ shuffled_data = self._load_shuffled_data(station, self.variables).reindex_like(hist)
+ return BootStrapGenerator(self.number_of_bootstraps, hist, shuffled_data, self.variables, variable)
+
+ def get_labels(self, station: str) -> np.ndarray:
+ """
+ Repeats labels for given key by the number of boots and returns as single array.
+ :param station: name of station
+ :return: repeated labels as single array
+ """
+ labels = self.data[station][1]
+ return np.tile(labels.data, (self.number_of_bootstraps, 1))
+
+ def get_orig_prediction(self, path: str, file_name: str, prediction_name: str = "CNN") -> np.ndarray:
+ """
+ Repeats predictions from given file(_name) in path by the number of boots.
+ :param path: path to file
+ :param file_name: file name
+ :param prediction_name: name of the prediction to select from loaded file (default CNN)
+ :return: repeated predictions
+ """
+ file = os.path.join(path, file_name)
+ prediction = xr.open_dataarray(file).sel(type=prediction_name).squeeze()
+ vals = np.tile(prediction.data, (self.number_of_bootstraps, 1))
+ return vals[~np.isnan(vals).any(axis=1), :]
+
+ def _load_shuffled_data(self, station: str, variables: List[str]) -> xr.DataArray:
+ """
+ Load shuffled data from bootstrap path. Data is stored as
+ '<station>_<var1>_<var2>_..._hist<histsize>_nboots<nboots>_shuffled.nc', e.g.
+ 'DEBW107_cloudcover_no_no2_temp_u_v_hist13_nboots20_shuffled.nc'
+ :param station: name of station
+ :param variables: list of variables
+ :return: shuffled data as xarray
+ """
+ file_name = self._get_shuffled_data_file(station, variables)
+ shuffled_data = xr.open_dataarray(file_name, chunks=100)
+ return shuffled_data
+
+ def _get_shuffled_data_file(self, station: str, variables: List[str]) -> str:
+ """
+ Looks for data file using regular expressions and returns found file or raise FileNotFoundError
+ :param station: name of station
+ :param variables: name of variables
+ :return: found file with complete path
+ """
+ files = os.listdir(self.bootstrap_path)
+ regex = self._create_file_regex(station, variables)
+ file = self._filter_files(regex, files, self.window_history_size, self.number_of_bootstraps)
+ if file:
+ return os.path.join(self.bootstrap_path, file)
+ else:
+ raise FileNotFoundError(f"Could not find a file to match pattern {regex}")
+
+ @staticmethod
+ def _create_file_regex(station: str, variables: List[str]) -> Pattern:
+ """
+ Creates regex for given station and variables to look for shuffled data with pattern:
+ `<station>(_<var>)*_hist(<hist>)_nboots(<nboots>)_shuffled.nc`
+ :param station: station name to use as prefix
+ :param variables: variables to add after station
+ :return: compiled regular expression
+ """
+ var_regex = "".join([rf"(_\w+)*_{v}(_\w+)*" for v in sorted(variables)])
+ regex = re.compile(rf"{station}{var_regex}_hist(\d+)_nboots(\d+)_shuffled\.nc")
+ return regex
+
+ @staticmethod
+ def _filter_files(regex: Pattern, files: List[str], window: int, nboot: int) -> Union[str, None]:
+ """
+ Filter list of files by regex. Regex has to be structured to match the following string structure
+ `<station>(_<var>)*_hist(<hist>)_nboots(<nboots>)_shuffled.nc`. Hist and nboots values have to be included as
+ group. All matches are compared to given window and nboot parameters. A valid file must have the same value (or
+ larger) than these parameters and contain all variables.
+ :param regex: compiled regular expression pattern following the style from method description
+ :param files: list of file names to filter
+ :param window: minimum length of window to look for
+ :param nboot: minimal number of boots to search
+ :return: matching file name or None, if no valid file was found
+ """
+ for f in files:
+ match = regex.match(f)
+ if match:
+ last = match.lastindex
+ if (int(match.group(last-1)) >= window) and (int(match.group(last)) >= nboot):
+ return f
+
+
if __name__ == "__main__":
from src.run_modules.experiment_setup import ExperimentSetup
diff --git a/src/data_handling/data_distributor.py b/src/data_handling/data_distributor.py
index b1624410e746ab779b20a60d6a7d19b4ae3b1267..e8c6044280799ded080ab4bff3627aeb9ffde2db 100644
--- a/src/data_handling/data_distributor.py
+++ b/src/data_handling/data_distributor.py
@@ -8,15 +8,18 @@ import math
import keras
import numpy as np
+from src.data_handling.data_generator import DataGenerator
+
class Distributor(keras.utils.Sequence):
- def __init__(self, generator: keras.utils.Sequence, model: keras.models, batch_size: int = 256,
- permute_data: bool = False):
+ def __init__(self, generator: DataGenerator, model: keras.models, batch_size: int = 256,
+ permute_data: bool = False, upsampling: bool = False):
self.generator = generator
self.model = model
self.batch_size = batch_size
self.do_data_permutation = permute_data
+ self.upsampling = upsampling
def _get_model_rank(self):
mod_out = self.model.output_shape
@@ -31,7 +34,7 @@ class Distributor(keras.utils.Sequence):
return mod_rank
def _get_number_of_mini_batches(self, values):
- return math.ceil(values[0].shape[0] / self.batch_size)
+ return math.ceil(values.shape[0] / self.batch_size)
def _permute_data(self, x, y):
"""
@@ -48,10 +51,18 @@ class Distributor(keras.utils.Sequence):
for k, v in enumerate(self.generator):
# get rank of output
mod_rank = self._get_model_rank()
- # get number of mini batches
- num_mini_batches = self._get_number_of_mini_batches(v)
+ # get data
x_total = np.copy(v[0])
y_total = np.copy(v[1])
+ if self.upsampling:
+ try:
+ s = self.generator.get_data_generator(k)
+ x_total = np.concatenate([x_total, np.copy(s.get_extremes_history())], axis=0)
+ y_total = np.concatenate([y_total, np.copy(s.get_extremes_label())], axis=0)
+ except AttributeError: # no extremes history / labels available, copy will fail
+ pass
+ # get number of mini batches
+ num_mini_batches = self._get_number_of_mini_batches(x_total)
# permute order for mini-batches
x_total, y_total = self._permute_data(x_total, y_total)
for prev, curr in enumerate(range(1, num_mini_batches+1)):
diff --git a/src/data_handling/data_generator.py b/src/data_handling/data_generator.py
index 24c9ada65b4bfd71de12785b2714cc5de94dc21f..8d10b3e438e185b9fd158259a6ba49a5612737be 100644
--- a/src/data_handling/data_generator.py
+++ b/src/data_handling/data_generator.py
@@ -14,6 +14,9 @@ from src import helpers
from src.data_handling.data_preparation import DataPrep
from src.join import EmptyQueryResult
+number = Union[float, int]
+num_or_list = Union[number, List[number]]
+
class DataGenerator(keras.utils.Sequence):
"""
@@ -27,7 +30,7 @@ class DataGenerator(keras.utils.Sequence):
def __init__(self, data_path: str, network: str, stations: Union[str, List[str]], variables: List[str],
interpolate_dim: str, target_dim: str, target_var: str, station_type: str = None,
interpolate_method: str = "linear", limit_nan_fill: int = 1, window_history_size: int = 7,
- window_lead_time: int = 4, transformation: Dict = None, **kwargs):
+ window_lead_time: int = 4, transformation: Dict = None, extreme_values: num_or_list = None, **kwargs):
self.data_path = os.path.abspath(data_path)
self.data_path_tmp = os.path.join(os.path.abspath(data_path), "tmp")
if not os.path.exists(self.data_path_tmp):
@@ -43,6 +46,7 @@ class DataGenerator(keras.utils.Sequence):
self.limit_nan_fill = limit_nan_fill
self.window_history_size = window_history_size
self.window_lead_time = window_lead_time
+ self.extreme_values = extreme_values
self.kwargs = kwargs
self.transformation = self.setup_transformation(transformation)
@@ -178,7 +182,7 @@ class DataGenerator(keras.utils.Sequence):
raise FileNotFoundError
data = self._load_pickle_data(station, self.variables)
except FileNotFoundError:
- logging.info(f"load not pickle data for {station}")
+ logging.debug(f"load not pickle data for {station}")
data = DataPrep(self.data_path, self.network, station, self.variables, station_type=self.station_type,
**self.kwargs)
if self.transformation is not None:
@@ -188,6 +192,9 @@ class DataGenerator(keras.utils.Sequence):
data.make_labels(self.target_dim, self.target_var, self.interpolate_dim, self.window_lead_time)
data.make_observation(self.target_dim, self.target_var, self.interpolate_dim)
data.remove_nan(self.interpolate_dim)
+ if self.extreme_values:
+ kwargs = {"extremes_on_right_tail_only": self.kwargs.get("extremes_on_right_tail_only", False)}
+ data.multiply_extremes(self.extreme_values, **kwargs)
if save_local_tmp_storage:
self._save_pickle_data(data)
return data
diff --git a/src/data_handling/data_preparation.py b/src/data_handling/data_preparation.py
index 3fae09306ab65d18f19d770b525cdc2296215bcd..5628394271918dc5631182d7de610db4ad335b7f 100644
--- a/src/data_handling/data_preparation.py
+++ b/src/data_handling/data_preparation.py
@@ -5,7 +5,7 @@ import datetime as dt
from functools import reduce
import logging
import os
-from typing import Union, List, Iterable
+from typing import Union, List, Iterable, Tuple
import numpy as np
import pandas as pd
@@ -17,6 +17,8 @@ from src import statistics
# define a more general date type for type hinting
date = Union[dt.date, dt.datetime]
str_or_list = Union[str, List[str]]
+number = Union[float, int]
+num_or_list = Union[number, List[number]]
class DataPrep(object):
@@ -58,6 +60,8 @@ class DataPrep(object):
self.history = None
self.label = None
self.observation = None
+ self.extremes_history = None
+ self.extremes_label = None
self.kwargs = kwargs
self.data = None
self.meta = None
@@ -353,7 +357,8 @@ class DataPrep(object):
non_nan_observation = self.observation.dropna(dim=dim)
intersect = reduce(np.intersect1d, (non_nan_history.coords[dim].values, non_nan_label.coords[dim].values, non_nan_observation.coords[dim].values))
- if len(intersect) == 0:
+ min_length = self.kwargs.get("min_length", 0)
+ if len(intersect) < max(min_length, 1):
self.history = None
self.label = None
self.observation = None
@@ -413,12 +418,79 @@ class DataPrep(object):
data.loc[..., used_chem_vars] = data.loc[..., used_chem_vars].clip(min=minimum)
return data
- def get_transposed_history(self):
+ def get_transposed_history(self) -> xr.DataArray:
return self.history.transpose("datetime", "window", "Stations", "variables").copy()
- def get_transposed_label(self):
+ def get_transposed_label(self) -> xr.DataArray:
return self.label.squeeze("Stations").transpose("datetime", "window").copy()
+ def get_extremes_history(self) -> xr.DataArray:
+ return self.extremes_history.transpose("datetime", "window", "Stations", "variables").copy()
+
+ def get_extremes_label(self):
+ return self.extremes_label.squeeze("Stations").transpose("datetime", "window").copy()
+
+ def multiply_extremes(self, extreme_values: num_or_list = 1., extremes_on_right_tail_only: bool = False,
+ timedelta: Tuple[int, str] = (1, 'm')):
+ """
+ This method extracts extreme values from self.labels which are defined in the argument extreme_values. One can
+ also decide only to extract extremes on the right tail of the distribution. When extreme_values is a list of
+ floats/ints all values larger (and smaller than negative extreme_values; extraction is performed in standardised
+ space) than are extracted iteratively. If for example extreme_values = [1.,2.] then a value of 1.5 would be
+ extracted once (for 0th entry in list), while a 2.5 would be extracted twice (once for each entry). Timedelta is
+ used to mark those extracted values by adding one min to each timestamp. As TOAR Data are hourly one can
+ identify those "artificial" data points later easily. Extreme inputs and labels are stored in
+ self.extremes_history and self.extreme_labels, respectively.
+
+ :param extreme_values: user definition of extreme
+ :param extremes_on_right_tail_only: if False also multiply values which are smaller then -extreme_values,
+ if True only extract values larger than extreme_values
+ :param timedelta: used as arguments for np.timedelta in order to mark extreme values on datetime
+ """
+
+ # check if labels or history is None
+ if (self.label is None) or (self.history is None):
+ logging.debug(f"{self.station} has `None' labels, skip multiply extremes")
+ return
+
+ # check type if inputs
+ extreme_values = helpers.to_list(extreme_values)
+ for i in extreme_values:
+ if not isinstance(i, number.__args__):
+ raise TypeError(f"Elements of list extreme_values have to be {number.__args__}, but at least element "
+ f"{i} is type {type(i)}")
+
+ for extr_val in sorted(extreme_values):
+ # check if some extreme values are already extracted
+ if (self.extremes_label is None) or (self.extremes_history is None):
+ # extract extremes based on occurance in labels
+ if extremes_on_right_tail_only:
+ extreme_label_idx = (self.label > extr_val).any(axis=0).values.reshape(-1,)
+ else:
+ extreme_label_idx = np.concatenate(((self.label < -extr_val).any(axis=0).values.reshape(-1, 1),
+ (self.label > extr_val).any(axis=0).values.reshape(-1, 1)),
+ axis=1).any(axis=1)
+ extremes_label = self.label[..., extreme_label_idx]
+ extremes_history = self.history[..., extreme_label_idx, :]
+ extremes_label.datetime.values += np.timedelta64(*timedelta)
+ extremes_history.datetime.values += np.timedelta64(*timedelta)
+ self.extremes_label = extremes_label#.squeeze('Stations').transpose('datetime', 'window')
+ self.extremes_history = extremes_history#.transpose('datetime', 'window', 'Stations', 'variables')
+ else: # one extr value iteration is done already: self.extremes_label is NOT None...
+ if extremes_on_right_tail_only:
+ extreme_label_idx = (self.extremes_label > extr_val).any(axis=0).values.reshape(-1, )
+ else:
+ extreme_label_idx = np.concatenate(((self.extremes_label < -extr_val).any(axis=0).values.reshape(-1, 1),
+ (self.extremes_label > extr_val).any(axis=0).values.reshape(-1, 1)
+ ), axis=1).any(axis=1)
+ # check on existing extracted extremes to minimise computational costs for comparison
+ extremes_label = self.extremes_label[..., extreme_label_idx]
+ extremes_history = self.extremes_history[..., extreme_label_idx, :]
+ extremes_label.datetime.values += np.timedelta64(*timedelta)
+ extremes_history.datetime.values += np.timedelta64(*timedelta)
+ self.extremes_label = xr.concat([self.extremes_label, extremes_label], dim='datetime')
+ self.extremes_history = xr.concat([self.extremes_history, extremes_history], dim='datetime')
+
if __name__ == "__main__":
dp = DataPrep('data/', 'dummy', 'DEBW107', ['o3', 'temp'], statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'})
diff --git a/src/datastore.py b/src/datastore.py
index d9f844ff97acb3f5c6600205f91100219d9c53e6..fb1650808a72f2a4d8b6afc10940cd9d14f894ba 100644
--- a/src/datastore.py
+++ b/src/datastore.py
@@ -3,6 +3,9 @@ __date__ = '2019-11-22'
from abc import ABC
+from functools import wraps
+import inspect
+import types
from typing import Any, List, Tuple, Dict
@@ -27,6 +30,57 @@ class EmptyScope(Exception):
pass
+class CorrectScope:
+ """
+ This class is used as decorator for all class methods, that have scope in parameters. After decoration, the scope
+ argument is not required on method call anymore. If no scope parameter is given, this decorator automatically adds
+ the default scope=`general` to the arguments. Furthermore, calls like `scope=general.sub` are obsolete, because this
+ decorator adds the prefix `general.` if not provided. Therefore, a call like `scope=sub` will actually become
+ `scope=general.sub` after passing this decorator.
+ """
+
+ def __init__(self, func):
+ wraps(func)(self)
+
+ def __call__(self, *args, **kwargs):
+ f_arg = inspect.getfullargspec(self.__wrapped__)
+ pos_scope = f_arg.args.index("scope")
+ if len(args) < (len(f_arg.args) - len(f_arg.defaults or "")):
+ new_arg = kwargs.pop("scope", "general") or "general"
+ args = self.update_tuple(args, new_arg, pos_scope)
+ else:
+ args = self.update_tuple(args, args[pos_scope], pos_scope, update=True)
+ return self.__wrapped__(*args, **kwargs)
+
+ def __get__(self, instance, cls):
+ return types.MethodType(self, instance)
+
+ @staticmethod
+ def correct(arg: str):
+ """
+ adds leading general prefix
+ :param arg: string argument of scope to add prefix general if necessary
+ :return: corrected string
+ """
+ if not arg.startswith("general"):
+ arg = "general." + arg
+ return arg
+
+ def update_tuple(self, t: Tuple, new: Any, ind: int, update: bool = False):
+ """
+ Either updates a entry in given tuple t (<old1>, <old2>, <old3>) --(ind=1)--> (<old1>, <new>, <old3>) or slots
+ entry into given position (<old1>, <old2>, <old3>) --(ind=1,update=True)--> (<old1>, <new>, <old2>, <old3>). In
+ the latter case, length of returned tuple is increased by 1 in comparison to given tuple.
+ :param t: tuple to update
+ :param new: new element to add to tuple
+ :param ind: position to add or slot in
+ :param update: updates entry if true, otherwise slot in (default: False)
+ :return: updated tuple
+ """
+ t_new = (*t[:ind], self.correct(new), *t[ind + update:])
+ return t_new
+
+
class AbstractDataStore(ABC):
"""
@@ -119,6 +173,7 @@ class DataStoreByVariable(AbstractDataStore):
<scope3>: value
"""
+ @CorrectScope
def set(self, name: str, obj: Any, scope: str) -> None:
"""
Store an object `obj` with given `name` under `scope`. In the current implementation, existing entries are
@@ -132,6 +187,7 @@ class DataStoreByVariable(AbstractDataStore):
self._store[name] = {}
self._store[name][scope] = obj
+ @CorrectScope
def get(self, name: str, scope: str) -> Any:
"""
Retrieve an object with `name` from `scope`. If no object can be found in the exact scope, take an iterative
@@ -144,6 +200,7 @@ class DataStoreByVariable(AbstractDataStore):
"""
return self._stride_through_scopes(name, scope)[2]
+ @CorrectScope
def get_default(self, name: str, scope: str, default: Any) -> Any:
"""
Same functionality like the standard get method. But this method adds a default argument that is returned if no
@@ -160,6 +217,7 @@ class DataStoreByVariable(AbstractDataStore):
except (NameNotFoundInDataStore, NameNotFoundInScope):
return default
+ @CorrectScope
def _stride_through_scopes(self, name, scope, depth=0):
if depth <= scope.count("."):
local_scope = scope.rsplit(".", maxsplit=depth)[0]
@@ -183,6 +241,7 @@ class DataStoreByVariable(AbstractDataStore):
"""
return sorted(self._store[name] if name in self._store.keys() else [])
+ @CorrectScope
def search_scope(self, scope: str, current_scope_only=True, return_all=False) -> List[str or Tuple]:
"""
Search for given `scope` and list all object names stored under this scope. To look also for all superior scopes
@@ -259,6 +318,7 @@ class DataStoreByScope(AbstractDataStore):
<variable3>: value
"""
+ @CorrectScope
def set(self, name: str, obj: Any, scope: str) -> None:
"""
Store an object `obj` with given `name` under `scope`. In the current implementation, existing entries are
@@ -271,6 +331,7 @@ class DataStoreByScope(AbstractDataStore):
self._store[scope] = {}
self._store[scope][name] = obj
+ @CorrectScope
def get(self, name: str, scope: str) -> Any:
"""
Retrieve an object with `name` from `scope`. If no object can be found in the exact scope, take an iterative
@@ -283,6 +344,7 @@ class DataStoreByScope(AbstractDataStore):
"""
return self._stride_through_scopes(name, scope)[2]
+ @CorrectScope
def get_default(self, name: str, scope: str, default: Any) -> Any:
"""
Same functionality like the standard get method. But this method adds a default argument that is returned if no
@@ -299,6 +361,7 @@ class DataStoreByScope(AbstractDataStore):
except (NameNotFoundInDataStore, NameNotFoundInScope):
return default
+ @CorrectScope
def _stride_through_scopes(self, name, scope, depth=0):
if depth <= scope.count("."):
local_scope = scope.rsplit(".", maxsplit=depth)[0]
@@ -326,6 +389,7 @@ class DataStoreByScope(AbstractDataStore):
keys.append(key)
return sorted(keys)
+ @CorrectScope
def search_scope(self, scope: str, current_scope_only: bool = True, return_all: bool = False) -> List[str or Tuple]:
"""
Search for given `scope` and list all object names stored under this scope. To look also for all superior scopes
diff --git a/src/helpers.py b/src/helpers.py
index 073a7bbf9ae3b7041591d48e4e5b7f3ef0efae42..be73614319b39dc36043437c64379342a96ce00e 100644
--- a/src/helpers.py
+++ b/src/helpers.py
@@ -4,17 +4,18 @@ __author__ = 'Lukas Leufen, Felix Kleinert'
__date__ = '2019-10-21'
+import datetime as dt
+from functools import wraps
import logging
import math
import os
-import time
import socket
-import datetime as dt
+import time
import keras.backend as K
import xarray as xr
-from typing import Dict, Callable
+from typing import Dict, Callable, Pattern, Union
def to_list(arg):
@@ -43,6 +44,16 @@ def l_p_loss(power: int):
return loss
+class TimeTrackingWrapper:
+
+ def __init__(self, func):
+ wraps(func)(self)
+
+ def __call__(self, *args, **kwargs):
+ with TimeTracking(name=self.__wrapped__.__name__):
+ return self.__wrapped__(*args, **kwargs)
+
+
class TimeTracking(object):
"""
Track time to measure execution time. Time tracking automatically starts on initialisation and ends by calling stop
@@ -81,7 +92,6 @@ class TimeTracking(object):
self._end()
else:
msg = f"Time was already stopped {time.time() - self.end}s ago."
- logging.error(msg)
raise AssertionError(msg)
if get_duration:
return self.duration()
@@ -99,6 +109,7 @@ class TimeTracking(object):
def prepare_host(create_new=True, sampling="daily"):
hostname = socket.gethostname()
+ runner_regex = re.compile(r"runner-.*-project-2411-concurrent-\d+")
try:
user = os.getlogin()
except OSError:
@@ -113,10 +124,9 @@ def prepare_host(create_new=True, sampling="daily"):
path = f"/p/project/cjjsc42/{user}/DATA/toar_{sampling}/"
elif (len(hostname) > 2) and (hostname[:2] == "jw"):
path = f"/p/home/jusers/{user}/juwels/intelliaq/DATA/toar_{sampling}/"
- elif "runner-6HmDp9Qd-project-2411-concurrent" in hostname:
+ elif runner_regex.match(hostname) is not None:
path = f"/home/{user}/machinelearningtools/data/toar_{sampling}/"
else:
- logging.error(f"unknown host '{hostname}'")
raise OSError(f"unknown host '{hostname}'")
if not os.path.exists(path):
try:
@@ -126,7 +136,6 @@ def prepare_host(create_new=True, sampling="daily"):
else:
raise PermissionError
except PermissionError:
- logging.error(f"path '{path}' does not exist for host '{hostname}'.")
raise NotADirectoryError(f"path '{path}' does not exist for host '{hostname}'.")
else:
logging.debug(f"set path to: {path}")
@@ -158,7 +167,7 @@ def set_bootstrap_path(bootstrap_path, data_path, sampling):
class PyTestRegex:
"""Assert that a given string meets some expectations."""
- def __init__(self, pattern: str, flags: int = 0):
+ def __init__(self, pattern: Union[str, Pattern], flags: int = 0):
self._regex = re.compile(pattern, flags)
def __eq__(self, actual: str) -> bool:
@@ -168,6 +177,28 @@ class PyTestRegex:
return self._regex.pattern
+class PyTestAllEqual:
+
+ def __init__(self, check_list):
+ self._list = check_list
+
+ def _check_all_equal(self):
+ equal = True
+ for b in self._list:
+ equal *= xr.testing.assert_equal(self._list[0], b) is None
+ return equal == 1
+
+ def is_true(self):
+ return self._check_all_equal()
+
+
+def xr_all_equal(check_list):
+ equal = True
+ for b in check_list:
+ equal *= xr.testing.assert_equal(check_list[0], b) is None
+ return equal == 1
+
+
def dict_to_xarray(d: Dict, coordinate_name: str) -> xr.DataArray:
"""
Convert a dictionary of 2D-xarrays to single 3D-xarray. The name of new coordinate axis follows <coordinate_name>.
@@ -200,21 +231,76 @@ def float_round(number: float, decimals: int = 0, round_type: Callable = math.ce
return round_type(number * multiplier) / multiplier
-def dict_pop(dict: Dict, pop_keys):
- pop_keys = to_list(pop_keys)
- return {k: v for k, v in dict.items() if k not in pop_keys}
-
-
def list_pop(list_full: list, pop_items):
pop_items = to_list(pop_items)
if len(pop_items) > 1:
return [e for e in list_full if e not in pop_items]
else:
- list_pop = list_full.copy()
- list_pop.remove(pop_items[0])
- return list_pop
+ l_pop = list_full.copy()
+ try:
+ l_pop.remove(pop_items[0])
+ except ValueError:
+ pass
+ return l_pop
def dict_pop(dict_orig: Dict, pop_keys):
pop_keys = to_list(pop_keys)
return {k: v for k, v in dict_orig.items() if k not in pop_keys}
+
+
+class Logger:
+ """
+ Basic logger class to unify all logging outputs. Logs are saved in local file and returned to std output. In default
+ settings, logging level of file logger is DEBUG, logging level of stream logger is INFO. Class must be imported
+ and initialised in starting script, all subscripts should log with logging.info(), debug, ...
+ """
+
+ def __init__(self, log_path=None, level_file=logging.DEBUG, level_stream=logging.INFO):
+
+ # define shared logger format
+ self.formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
+
+ # set log path
+ self.log_file = self.setup_logging_path(log_path)
+ # set root logger as file handler
+ logging.basicConfig(level=level_file,
+ format=self.formatter,
+ filename=self.log_file,
+ filemode='a')
+ # add stream handler to the root logger
+ logging.getLogger('').addHandler(self.logger_console(level_stream))
+ # print logger path
+ logging.info(f"File logger: {self.log_file}")
+
+ @staticmethod
+ def setup_logging_path(path: str = None):
+ """
+ Check if given path exists and creates if not. If path is None, use path from main. The logging file is named
+ like `logging_<runtime>.log` where runtime=`%Y-%m-%d_%H-%M-%S` of current run.
+ :param path: path to logfile
+ :return: path of logfile
+ """
+ if not path: # set default path
+ path = os.path.join(os.path.dirname(__file__), "..", "logging")
+ if not os.path.exists(path):
+ os.makedirs(path)
+ runtime = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime())
+ log_file = os.path.join(path, f'logging_{runtime}.log')
+ return log_file
+
+ def logger_console(self, level: int):
+ """
+ Defines a stream handler which writes messages of given level or higher to std out
+ :param level: logging level as integer, e.g. logging.DEBUG or 10
+ :return: defines stream handler
+ """
+ # define Handler
+ console = logging.StreamHandler()
+ # set level of Handler
+ console.setLevel(level)
+ # set a format which is simpler for console use
+ formatter = logging.Formatter(self.formatter)
+ # tell the handler to use this format
+ console.setFormatter(formatter)
+ return console
diff --git a/src/model_modules/advanced_paddings.py b/src/model_modules/advanced_paddings.py
index 1d48dfc0a87c05183fc5b8b7755f48efaf7b5428..d9e55c78fb6c78bbe219c820078c46a235627897 100644
--- a/src/model_modules/advanced_paddings.py
+++ b/src/model_modules/advanced_paddings.py
@@ -6,6 +6,7 @@ import numpy as np
import keras.backend as K
from keras.layers.convolutional import _ZeroPadding
+from keras.layers import ZeroPadding2D
from keras.legacy import interfaces
from keras.utils import conv_utils
from keras.utils.generic_utils import transpose_shape
@@ -252,13 +253,50 @@ class SymmetricPadding2D(_ZeroPadding):
return tf.pad(inputs, pattern, 'SYMMETRIC')
+class Padding2D:
+ '''
+ This class combines the implemented padding methods. You can call this method by defining a specific padding type.
+ The __call__ method will return the corresponding Padding layer.
+ '''
+
+ allowed_paddings = {
+ **dict.fromkeys(("RefPad2D", "ReflectionPadding2D"), ReflectionPadding2D),
+ **dict.fromkeys(("SymPad2D", "SymmetricPadding2D"), SymmetricPadding2D),
+ **dict.fromkeys(("ZeroPad2D", "ZeroPadding2D"), ZeroPadding2D)
+ }
+
+ def __init__(self, padding_type):
+ self.padding_type = padding_type
+
+ def _check_and_get_padding(self):
+ if isinstance(self.padding_type, str):
+ try:
+ pad2d = self.allowed_paddings[self.padding_type]
+ except KeyError as einfo:
+ raise NotImplementedError(
+ f"`{einfo}' is not implemented as padding. "
+ "Use one of those: i) `RefPad2D', ii) `SymPad2D', iii) `ZeroPad2D'")
+ else:
+ if self.padding_type in self.allowed_paddings.values():
+ pad2d = self.padding_type
+ else:
+ raise TypeError(f"`{self.padding_type.__name__}' is not a valid padding layer type. "
+ "Use one of those: "
+ "i) ReflectionPadding2D, ii) SymmetricPadding2D, iii) ZeroPadding2D")
+ return pad2d
+
+ def __call__(self, *args, **kwargs):
+ return self._check_and_get_padding()(*args, **kwargs)
+
+
if __name__ == '__main__':
from keras.models import Model
from keras.layers import Conv2D, Flatten, Dense, Input
kernel_1 = (3, 3)
kernel_2 = (5, 5)
- x = np.array(range(2000)).reshape(-1, 10, 10, 1)
+ kernel_3 = (3, 3)
+ x = np.array(range(2000)).reshape((-1, 10, 10, 1))
y = x.mean(axis=(1, 2))
x_input = Input(shape=x.shape[1:])
@@ -269,6 +307,10 @@ if __name__ == '__main__':
pad2 = PadUtils.get_padding_for_same(kernel_size=kernel_2)
x_out = SymmetricPadding2D(padding=pad2, name="SymPAD")(x_out)
x_out = Conv2D(2, kernel_size=kernel_2, activation='relu')(x_out)
+
+ pad3 = PadUtils.get_padding_for_same(kernel_size=kernel_3)
+ x_out = Padding2D('RefPad2D')(padding=pad3, name="Padding2D_RefPad")(x_out)
+ x_out = Conv2D(2, kernel_size=kernel_3, activation='relu')(x_out)
x_out = Flatten()(x_out)
x_out = Dense(1, activation='linear')(x_out)
diff --git a/src/model_modules/inception_model.py b/src/model_modules/inception_model.py
index 1cb7656335495f0261abb434e4a203cb4e63887e..15739556d7d28d9e7e6ecc454615d82fb81a2754 100644
--- a/src/model_modules/inception_model.py
+++ b/src/model_modules/inception_model.py
@@ -5,7 +5,7 @@ import logging
import keras
import keras.layers as layers
-from src.model_modules.advanced_paddings import PadUtils, ReflectionPadding2D, SymmetricPadding2D
+from src.model_modules.advanced_paddings import PadUtils, ReflectionPadding2D, SymmetricPadding2D, Padding2D
class InceptionModelBase:
@@ -75,9 +75,9 @@ class InceptionModelBase:
name=f'Block_{self.number_of_blocks}{self.block_part_name()}_1x1')(input_x)
tower = self.act(tower, activation, **act_settings)
- tower = self.padding_layer(padding)(padding=padding_size,
- name=f'Block_{self.number_of_blocks}{self.block_part_name()}_Pad'
- )(tower)
+ tower = Padding2D(padding)(padding=padding_size,
+ name=f'Block_{self.number_of_blocks}{self.block_part_name()}_Pad'
+ )(tower)
tower = layers.Conv2D(tower_filter,
tower_kernel,
@@ -108,29 +108,6 @@ class InceptionModelBase:
else:
return act_name.__name__
- @staticmethod
- def padding_layer(padding):
- allowed_paddings = {
- 'RefPad2D': ReflectionPadding2D, 'ReflectionPadding2D': ReflectionPadding2D,
- 'SymPad2D': SymmetricPadding2D, 'SymmetricPadding2D': SymmetricPadding2D,
- 'ZeroPad2D': keras.layers.ZeroPadding2D, 'ZeroPadding2D': keras.layers.ZeroPadding2D
- }
- if isinstance(padding, str):
- try:
- pad2d = allowed_paddings[padding]
- except KeyError as einfo:
- raise NotImplementedError(
- f"`{einfo}' is not implemented as padding. "
- "Use one of those: i) `RefPad2D', ii) `SymPad2D', iii) `ZeroPad2D'")
- else:
- if padding in allowed_paddings.values():
- pad2d = padding
- else:
- raise TypeError(f"`{padding.__name__}' is not a valid padding layer type. "
- "Use one of those: "
- "i) ReflectionPadding2D, ii) SymmetricPadding2D, iii) ZeroPadding2D")
- return pad2d
-
def create_pool_tower(self, input_x, pool_kernel, tower_filter, activation='relu', max_pooling=True, **kwargs):
"""
This function creates a "MaxPooling tower block"
@@ -156,7 +133,7 @@ class InceptionModelBase:
block_type = "AvgPool"
pooling = layers.AveragePooling2D
- tower = self.padding_layer(padding)(padding=padding_size, name=block_name+'Pad')(input_x)
+ tower = Padding2D(padding)(padding=padding_size, name=block_name+'Pad')(input_x)
tower = pooling(pool_kernel, strides=(1, 1), padding='valid', name=block_name+block_type)(tower)
# convolution block
@@ -211,35 +188,6 @@ class InceptionModelBase:
return block
-# if __name__ == '__main__':
-# from keras.models import Model
-# from keras.layers import Conv2D, Flatten, Dense, Input
-# import numpy as np
-#
-#
-# kernel_1 = (3, 3)
-# kernel_2 = (5, 5)
-# x = np.array(range(2000)).reshape(-1, 10, 10, 1)
-# y = x.mean(axis=(1, 2))
-#
-# x_input = Input(shape=x.shape[1:])
-# pad1 = PadUtils.get_padding_for_same(kernel_size=kernel_1)
-# x_out = InceptionModelBase.padding_layer('RefPad2D')(padding=pad1, name="RefPAD1")(x_input)
-# # x_out = ReflectionPadding2D(padding=pad1, name="RefPAD")(x_input)
-# x_out = Conv2D(5, kernel_size=kernel_1, activation='relu')(x_out)
-#
-# pad2 = PadUtils.get_padding_for_same(kernel_size=kernel_2)
-# x_out = InceptionModelBase.padding_layer(SymmetricPadding2D)(padding=pad2, name="SymPAD1")(x_out)
-# # x_out = SymmetricPadding2D(padding=pad2, name="SymPAD")(x_out)
-# x_out = Conv2D(2, kernel_size=kernel_2, activation='relu')(x_out)
-# x_out = Flatten()(x_out)
-# x_out = Dense(1, activation='linear')(x_out)
-#
-# model = Model(inputs=x_input, outputs=x_out)
-# model.compile('adam', loss='mse')
-# model.summary()
-# # model.fit(x, y, epochs=10)
-
if __name__ == '__main__':
print(__name__)
from keras.datasets import cifar10
@@ -262,7 +210,7 @@ if __name__ == '__main__':
'padding': 'SymPad2D'},
'tower_3': {'reduction_filter': 64,
'tower_filter': 64,
- 'tower_kernel': (1, 1),
+ 'tower_kernel': (7, 7),
'activation': ELU,
'padding': ReflectionPadding2D}
}
diff --git a/src/model_modules/model_class.py b/src/model_modules/model_class.py
index ebbd7a25cef9031436d932a6502c9726bfe3e318..d6dcea179bcfa8a6ec41518db34b186e30d908fc 100644
--- a/src/model_modules/model_class.py
+++ b/src/model_modules/model_class.py
@@ -5,11 +5,12 @@ __date__ = '2019-12-12'
from abc import ABC
-from typing import Any, Callable
+from typing import Any, Callable, Dict
import keras
from src.model_modules.inception_model import InceptionModelBase
from src.model_modules.flatten import flatten_tail
+from src.model_modules.advanced_paddings import PadUtils, Padding2D
class AbstractModelClass(ABC):
@@ -30,6 +31,7 @@ class AbstractModelClass(ABC):
self.__model = None
self.__loss = None
self.model_name = self.__class__.__name__
+ self.__custom_objects = {}
def __getattr__(self, name: str) -> Any:
@@ -78,9 +80,44 @@ class AbstractModelClass(ABC):
def loss(self, value) -> None:
self.__loss = value
- def get_settings(self):
+ @property
+ def custom_objects(self) -> Dict:
+ """
+ The custom objects property collects all non-keras utilities that are used in the model class. To load such a
+ customised and already compiled model (e.g. from local disk), this information is required.
+ :return: the custom objects in a dictionary
+ """
+ return self.__custom_objects
+
+ @custom_objects.setter
+ def custom_objects(self, value) -> None:
+ self.__custom_objects = value
+
+ def get_settings(self) -> Dict:
+ """
+ Get all class attributes that are not protected in the AbstractModelClass as dictionary.
+ :return: all class attributes
+ """
return dict((k, v) for (k, v) in self.__dict__.items() if not k.startswith("_AbstractModelClass__"))
+ def set_model(self):
+ pass
+
+ def set_loss(self):
+ pass
+
+ def set_custom_objects(self, **kwargs) -> None:
+ """
+ Set custom objects that are not part of keras framework. These custom objects are needed if an already compiled
+ model is loaded from disk. There is a special treatment for the Padding2D class, which is a base class for
+ different padding types. For a correct behaviour, all supported subclasses are added as custom objects in
+ addition to the given ones.
+ :param kwargs: all custom objects, that should be saved
+ """
+ if "Padding2D" in kwargs.keys():
+ kwargs.update(kwargs["Padding2D"].allowed_paddings)
+ self.custom_objects = kwargs
+
class MyLittleModel(AbstractModelClass):
@@ -120,6 +157,7 @@ class MyLittleModel(AbstractModelClass):
# apply to model
self.set_model()
self.set_loss()
+ self.set_custom_objects(loss=self.loss)
def set_model(self):
@@ -200,6 +238,7 @@ class MyBranchedModel(AbstractModelClass):
# apply to model
self.set_model()
self.set_loss()
+ self.set_custom_objects(loss=self.loss)
def set_model(self):
@@ -276,6 +315,7 @@ class MyTowerModel(AbstractModelClass):
# apply to model
self.set_model()
self.set_loss()
+ self.set_custom_objects(loss=self.loss)
def set_model(self):
@@ -351,3 +391,137 @@ class MyTowerModel(AbstractModelClass):
"""
self.loss = [keras.losses.mean_squared_error]
+
+
+class MyPaperModel(AbstractModelClass):
+
+ def __init__(self, window_history_size, window_lead_time, channels):
+
+ """
+ Sets model and loss depending on the given arguments.
+ :param activation: activation function
+ :param window_history_size: number of historical time steps included in the input data
+ :param channels: number of variables used in input data
+ :param regularizer: <not used here>
+ :param dropout_rate: dropout rate used in the model [0, 1)
+ :param window_lead_time: number of time steps to forecast in the output layer
+ """
+
+ super().__init__()
+
+ # settings
+ self.window_history_size = window_history_size
+ self.window_lead_time = window_lead_time
+ self.channels = channels
+ self.dropout_rate = .3
+ self.regularizer = keras.regularizers.l2(0.001)
+ self.initial_lr = 1e-3
+ # self.optimizer = keras.optimizers.adam(lr=self.initial_lr, amsgrad=True)
+ self.optimizer = keras.optimizers.SGD(lr=self.initial_lr, momentum=0.9)
+ self.lr_decay = src.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94, epochs_drop=10)
+ self.epochs = 150
+ self.batch_size = int(256 * 2)
+ self.activation = keras.layers.ELU
+ self.padding = "SymPad2D"
+
+ # apply to model
+ self.set_model()
+ self.set_loss()
+ self.set_custom_objects(loss=self.loss, Padding2D=Padding2D)
+
+ def set_model(self):
+
+ """
+ Build the model.
+ :param activation: activation function
+ :param window_history_size: number of historical time steps included in the input data
+ :param channels: number of variables used in input data
+ :param dropout_rate: dropout rate used in the model [0, 1)
+ :param window_lead_time: number of time steps to forecast in the output layer
+ :return: built keras model
+ """
+ activation = self.activation
+ first_kernel = (3,1)
+ first_filters = 16
+
+ conv_settings_dict1 = {
+ 'tower_1': {'reduction_filter': 8, 'tower_filter': 16 * 2, 'tower_kernel': (3, 1),
+ 'activation': activation},
+ 'tower_2': {'reduction_filter': 8, 'tower_filter': 16 * 2, 'tower_kernel': (5, 1),
+ 'activation': activation},
+ 'tower_3': {'reduction_filter': 8, 'tower_filter': 16 * 2, 'tower_kernel': (1, 1),
+ 'activation': activation},
+ # 'tower_4':{'reduction_filter':8, 'tower_filter':8*2, 'tower_kernel':(7,1), 'activation':activation},
+ }
+ pool_settings_dict1 = {'pool_kernel': (3, 1), 'tower_filter': 16, 'activation': activation}
+
+ conv_settings_dict2 = {
+ 'tower_1': {'reduction_filter': 64, 'tower_filter': 32 * 2, 'tower_kernel': (3, 1),
+ 'activation': activation},
+ 'tower_2': {'reduction_filter': 64, 'tower_filter': 32 * 2, 'tower_kernel': (5, 1),
+ 'activation': activation},
+ 'tower_3': {'reduction_filter': 64, 'tower_filter': 32 * 2, 'tower_kernel': (1, 1),
+ 'activation': activation},
+ # 'tower_4':{'reduction_filter':8*2, 'tower_filter':16*2, 'tower_kernel':(7,1), 'activation':activation},
+ }
+ pool_settings_dict2 = {'pool_kernel': (3, 1), 'tower_filter': 32, 'activation': activation}
+
+ conv_settings_dict3 = {
+ 'tower_1': {'reduction_filter': 64 * 2, 'tower_filter': 32 * 4, 'tower_kernel': (3, 1),
+ 'activation': activation},
+ 'tower_2': {'reduction_filter': 64 * 2, 'tower_filter': 32 * 4, 'tower_kernel': (5, 1),
+ 'activation': activation},
+ 'tower_3': {'reduction_filter': 64 * 2, 'tower_filter': 32 * 4, 'tower_kernel': (1, 1),
+ 'activation': activation},
+ # 'tower_4':{'reduction_filter':16*4, 'tower_filter':32, 'tower_kernel':(7,1), 'activation':activation},
+ }
+ pool_settings_dict3 = {'pool_kernel': (3, 1), 'tower_filter': 32, 'activation': activation}
+
+ ##########################################
+ inception_model = InceptionModelBase()
+
+ X_input = keras.layers.Input(
+ shape=(self.window_history_size + 1, 1, self.channels)) # add 1 to window_size to include current time step t0
+
+ pad_size = PadUtils.get_padding_for_same(first_kernel)
+ # X_in = adv_pad.SymmetricPadding2D(padding=pad_size)(X_input)
+ # X_in = inception_model.padding_layer("SymPad2D")(padding=pad_size, name="SymPad")(X_input) # adv_pad.SymmetricPadding2D(padding=pad_size)(X_input)
+ X_in = Padding2D("SymPad2D")(padding=pad_size, name="SymPad")(X_input)
+ X_in = keras.layers.Conv2D(filters=first_filters,
+ kernel_size=first_kernel,
+ kernel_regularizer=self.regularizer,
+ name="First_conv_{}x{}".format(first_kernel[0], first_kernel[1]))(X_in)
+ X_in = self.activation(name='FirstAct')(X_in)
+
+
+ X_in = inception_model.inception_block(X_in, conv_settings_dict1, pool_settings_dict1,
+ regularizer=self.regularizer,
+ batch_normalisation=True,
+ padding=self.padding)
+ out_minor1 = flatten_tail(X_in, 'minor_1', False, self.dropout_rate, self.window_lead_time,
+ self.activation, 32, 64)
+
+ X_in = keras.layers.Dropout(self.dropout_rate)(X_in)
+
+ X_in = inception_model.inception_block(X_in, conv_settings_dict2, pool_settings_dict2, regularizer=self.regularizer,
+ batch_normalisation=True, padding=self.padding)
+
+ # X_in = keras.layers.Dropout(self.dropout_rate)(X_in)
+ #
+ # X_in = inception_model.inception_block(X_in, conv_settings_dict3, pool_settings_dict3, regularizer=self.regularizer,
+ # batch_normalisation=True)
+ #############################################
+
+ out_main = flatten_tail(X_in, 'Main', activation=activation, bound_weight=False, dropout_rate=self.dropout_rate,
+ reduction_filter=64 * 2, first_dense=64 * 2, window_lead_time=self.window_lead_time)
+
+ self.model = keras.Model(inputs=X_input, outputs=[out_minor1, out_main])
+
+ def set_loss(self):
+
+ """
+ Set the loss
+ :return: loss function
+ """
+
+ self.loss = [keras.losses.mean_squared_error, keras.losses.mean_squared_error]
diff --git a/src/plotting/postprocessing_plotting.py b/src/plotting/postprocessing_plotting.py
index 854182613cdb63456dc8f62d2421560d829ee629..14e3074a7d8f09bd597fb2fbf53a298d83ab6556 100644
--- a/src/plotting/postprocessing_plotting.py
+++ b/src/plotting/postprocessing_plotting.py
@@ -16,15 +16,37 @@ import pandas as pd
import seaborn as sns
import xarray as xr
from matplotlib.backends.backend_pdf import PdfPages
+import matplotlib.patches as mpatches
from src import helpers
-from src.helpers import TimeTracking
-from src.run_modules.run_environment import RunEnvironment
+from src.helpers import TimeTracking, TimeTrackingWrapper
+from src.data_handling.data_generator import DataGenerator
logging.getLogger('matplotlib').setLevel(logging.WARNING)
-class PlotMonthlySummary(RunEnvironment):
+class AbstractPlotClass:
+
+ def __init__(self, plot_folder, plot_name, resolution=500):
+ self.plot_folder = plot_folder
+ self.plot_name = plot_name
+ self.resolution = resolution
+
+ def _plot(self, *args):
+ raise NotImplementedError
+
+ def _save(self, **kwargs):
+ """
+ Standard save method to store plot locally. Name of and path to plot need to be set on initialisation
+ """
+ plot_name = os.path.join(os.path.abspath(self.plot_folder), f"{self.plot_name}.pdf")
+ logging.debug(f"... save plot to {plot_name}")
+ plt.savefig(plot_name, dpi=self.resolution, **kwargs)
+ plt.close('all')
+
+
+@TimeTrackingWrapper
+class PlotMonthlySummary(AbstractPlotClass):
"""
Show a monthly summary over all stations for each lead time ("ahead") as box and whiskers plot. The plot is saved
in data_path with name monthly_summary_box_plot.pdf and 500dpi resolution.
@@ -41,12 +63,13 @@ class PlotMonthlySummary(RunEnvironment):
the maximum lead time from data is used. (default None -> use maximum lead time from data).
:param plot_folder: path to save the plot (default: current directory)
"""
- super().__init__()
+ super().__init__(plot_folder, "monthly_summary_box_plot")
self._data_path = data_path
self._data_name = name
self._data = self._prepare_data(stations)
self._window_lead_time = self._get_window_lead_time(window_lead_time)
- self._plot(target_var, plot_folder)
+ self._plot(target_var)
+ self._save()
def _prepare_data(self, stations: List) -> xr.DataArray:
"""
@@ -90,12 +113,11 @@ class PlotMonthlySummary(RunEnvironment):
window_lead_time = ahead_steps
return min(ahead_steps, window_lead_time)
- def _plot(self, target_var: str, plot_folder: str):
+ def _plot(self, target_var: str):
"""
Main plot function that creates a monthly grouped box plot over all stations but with separate boxes for each
lead time step.
:param target_var: display name of the target variable on plot's axis
- :param plot_folder: path to save the plot
"""
data = self._data.to_dataset(name='values').to_dask_dataframe()
logging.debug("... start plotting")
@@ -105,21 +127,10 @@ class PlotMonthlySummary(RunEnvironment):
meanprops={'markersize': 1, 'markeredgecolor': 'k'})
ax.set(xlabel='month', ylabel=f'{target_var}')
plt.tight_layout()
- self._save(plot_folder)
-
- @staticmethod
- def _save(plot_folder):
- """
- Standard save method to store plot locally. The name of this plot is static.
- :param plot_folder: path to save the plot
- """
- plot_name = os.path.join(os.path.abspath(plot_folder), 'monthly_summary_box_plot.pdf')
- logging.debug(f"... save plot to {plot_name}")
- plt.savefig(plot_name, dpi=500)
- plt.close('all')
-class PlotStationMap(RunEnvironment):
+@TimeTrackingWrapper
+class PlotStationMap(AbstractPlotClass):
"""
Plot geographical overview of all used stations as squares. Different data sets can be colorised by its key in the
input dictionary generators. The key represents the color to plot on the map. Currently, there is only a white
@@ -133,9 +144,10 @@ class PlotStationMap(RunEnvironment):
as value.
:param plot_folder: path to save the plot (default: current directory)
"""
- super().__init__()
+ super().__init__(plot_folder, "station_map")
self._ax = None
- self._plot(generators, plot_folder)
+ self._plot(generators)
+ self._save()
def _draw_background(self):
"""
@@ -163,32 +175,20 @@ class PlotStationMap(RunEnvironment):
station_coords.loc['station_lat'].values)
self._ax.plot(IDx, IDy, mfc=color, mec='k', marker='s', markersize=6, transform=ccrs.PlateCarree())
- def _plot(self, generators: Dict, plot_folder: str):
+ def _plot(self, generators: Dict):
"""
Main plot function to create the station map plot. Sets figure and calls all required sub-methods.
:param generators: dictionary with the plot color of each data set as key and the generator containing all
stations as value.
- :param plot_folder: path to save the plot
"""
fig = plt.figure(figsize=(10, 5))
self._ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
self._ax.set_extent([0, 20, 42, 58], crs=ccrs.PlateCarree())
self._draw_background()
self._plot_stations(generators)
- self._save(plot_folder)
-
- @staticmethod
- def _save(plot_folder):
- """
- Standard save method to store plot locally. The name of this plot is static.
- :param plot_folder: path to save the plot
- """
- plot_name = os.path.join(os.path.abspath(plot_folder), 'station_map.pdf')
- logging.debug(f"... save plot to {plot_name}")
- plt.savefig(plot_name, dpi=500)
- plt.close('all')
+@TimeTrackingWrapper
def plot_conditional_quantiles(stations: list, plot_folder: str = ".", rolling_window: int = 3, ref_name: str = 'obs',
pred_name: str = 'CNN', season: str = "", forecast_path: str = None,
plot_name_affix: str = "", units: str = "ppb"):
@@ -207,7 +207,7 @@ def plot_conditional_quantiles(stations: list, plot_folder: str = ".", rolling_w
:param plot_name_affix: name to specify this plot (e.g. 'cali-ref', default: '')
:param units: units of the forecasted values (default: ppb)
"""
- time = TimeTracking()
+ # time = TimeTracking()
logging.debug(f"started plot_conditional_quantiles()")
# ignore warnings if nans appear in quantile grouping
warnings.filterwarnings("ignore", message="All-NaN slice encountered")
@@ -321,10 +321,11 @@ def plot_conditional_quantiles(stations: list, plot_folder: str = ".", rolling_w
# close all open figures / plots
pdf_pages.close()
plt.close('all')
- logging.info(f"plot_conditional_quantiles() finished after {time}")
+ #logging.info(f"plot_conditional_quantiles() finished after {time}")
-class PlotClimatologicalSkillScore(RunEnvironment):
+@TimeTrackingWrapper
+class PlotClimatologicalSkillScore(AbstractPlotClass):
"""
Create plot of climatological skill score after Murphy (1988) as box plot over all stations. A forecast time step
(called "ahead") is separately shown to highlight the differences for each prediction time step. Either each single
@@ -342,10 +343,11 @@ class PlotClimatologicalSkillScore(RunEnvironment):
:param extra_name_tag: additional tag that can be included in the plot name (default "")
:param model_setup: architecture type to specify plot name (default "CNN")
"""
- super().__init__()
+ super().__init__(plot_folder, f"skill_score_clim_{extra_name_tag}{model_setup}")
self._labels = None
self._data = self._prepare_data(data, score_only)
- self._plot(plot_folder, score_only, extra_name_tag, model_setup)
+ self._plot(score_only)
+ self._save()
def _prepare_data(self, data: Dict, score_only: bool) -> pd.DataFrame:
"""
@@ -369,13 +371,10 @@ class PlotClimatologicalSkillScore(RunEnvironment):
"""
return "" if score_only else "terms and "
- def _plot(self, plot_folder, score_only, extra_name_tag, model_setup):
+ def _plot(self, score_only):
"""
Main plot function to plot climatological skill score.
- :param plot_folder: path to save the plot
:param score_only: if true plot only scores of CASE I to IV, otherwise plot all single terms
- :param extra_name_tag: additional tag that can be included in the plot name
- :param model_setup: architecture type to specify plot name
"""
fig, ax = plt.subplots()
if not score_only:
@@ -387,24 +386,10 @@ class PlotClimatologicalSkillScore(RunEnvironment):
handles, _ = ax.get_legend_handles_labels()
ax.legend(handles, self._labels)
plt.tight_layout()
- self._save(plot_folder, extra_name_tag, model_setup)
-
- @staticmethod
- def _save(plot_folder, extra_name_tag, model_setup):
- """
- Standard save method to store plot locally. The name of this plot is dynamic. It includes the model setup like
- 'CNN' and can additionally be adjusted using an extra name tag.
- :param plot_folder: path to save the plot
- :param extra_name_tag: additional tag that can be included in the plot name
- :param model_setup: architecture type to specify plot name
- """
- plot_name = os.path.join(plot_folder, f"skill_score_clim_{extra_name_tag}{model_setup}.pdf")
- logging.debug(f"... save plot to {plot_name}")
- plt.savefig(plot_name, dpi=500)
- plt.close('all')
-class PlotCompetitiveSkillScore(RunEnvironment):
+@TimeTrackingWrapper
+class PlotCompetitiveSkillScore(AbstractPlotClass):
"""
Create competitive skill score for the given model setup and the reference models ordinary least squared ("ols") and
the persistence forecast ("persi") for all lead times ("ahead"). The plot is saved under plot_folder with the name
@@ -417,10 +402,11 @@ class PlotCompetitiveSkillScore(RunEnvironment):
:param plot_folder: path to save the plot (default: current directory)
:param model_setup: architecture type (default "CNN")
"""
- super().__init__()
+ super().__init__(plot_folder, f"skill_score_competitive_{model_setup}")
self._labels = None
self._data = self._prepare_data(data)
- self._plot(plot_folder, model_setup)
+ self._plot()
+ self._save()
def _prepare_data(self, data: pd.DataFrame) -> pd.DataFrame:
"""
@@ -437,12 +423,9 @@ class PlotCompetitiveSkillScore(RunEnvironment):
self._labels = [str(i) + "d" for i in data.index.levels[1].values]
return data.stack(level=0).reset_index(level=2, drop=True).reset_index(name="data")
- def _plot(self, plot_folder, model_setup):
+ def _plot(self):
"""
Main plot function to plot skill scores of the comparisons cnn-persi, ols-persi and cnn-ols.
- :param plot_folder: path to save the plot
- :param model_setup:
- :return: architecture type to specify plot name
"""
fig, ax = plt.subplots()
sns.boxplot(x="comparison", y="data", hue="ahead", data=self._data, whis=1., ax=ax, palette="Blues_d",
@@ -454,7 +437,6 @@ class PlotCompetitiveSkillScore(RunEnvironment):
handles, _ = ax.get_legend_handles_labels()
ax.legend(handles, self._labels)
plt.tight_layout()
- self._save(plot_folder, model_setup)
def _ylim(self) -> Tuple[float, float]:
"""
@@ -466,20 +448,9 @@ class PlotCompetitiveSkillScore(RunEnvironment):
upper = helpers.float_round(self._data.max()[2], 2) + 0.1
return lower, upper
- @staticmethod
- def _save(plot_folder, model_setup):
- """
- Standard save method to store plot locally. The name of this plot is dynamic by including the model setup.
- :param plot_folder: path to save the plot
- :param model_setup: architecture type to specify plot name
- """
- plot_name = os.path.join(plot_folder, f"skill_score_competitive_{model_setup}.pdf")
- logging.debug(f"... save plot to {plot_name}")
- plt.savefig(plot_name, dpi=500)
- plt.close()
-
-class PlotBootstrapSkillScore(RunEnvironment):
+@TimeTrackingWrapper
+class PlotBootstrapSkillScore(AbstractPlotClass):
"""
Create plot of climatological skill score after Murphy (1988) as box plot over all stations. A forecast time step
(called "ahead") is separately shown to highlight the differences for each prediction time step. Either each single
@@ -495,10 +466,12 @@ class PlotBootstrapSkillScore(RunEnvironment):
:param plot_folder: path to save the plot (default: current directory)
:param model_setup: architecture type to specify plot name (default "CNN")
"""
- super().__init__()
+ super().__init__(plot_folder, f"skill_score_bootstrap_{model_setup}")
self._labels = None
+ self._x_name = "boot_var"
self._data = self._prepare_data(data)
- self._plot(plot_folder, model_setup)
+ self._plot()
+ self._save()
def _prepare_data(self, data: Dict) -> pd.DataFrame:
"""
@@ -507,7 +480,7 @@ class PlotBootstrapSkillScore(RunEnvironment):
:param data: dictionary with station names as keys and 2D xarrays as values
:return: pre-processed data set
"""
- data = helpers.dict_to_xarray(data, "station")
+ data = helpers.dict_to_xarray(data, "station").sortby(self._x_name)
self._labels = [str(i) + "d" for i in data.coords["ahead"].values]
return data.to_dataframe("data").reset_index(level=[0, 1, 2])
@@ -519,41 +492,25 @@ class PlotBootstrapSkillScore(RunEnvironment):
"""
return "" if score_only else "terms and "
- def _plot(self, plot_folder, model_setup):
+ def _plot(self):
"""
Main plot function to plot climatological skill score.
- :param plot_folder: path to save the plot
- :param model_setup: architecture type to specify plot name
"""
fig, ax = plt.subplots()
- sns.boxplot(x="boot_var", y="data", hue="ahead", data=self._data, ax=ax, whis=1., palette="Blues_d",
+ sns.boxplot(x=self._x_name, y="data", hue="ahead", data=self._data, ax=ax, whis=1., palette="Blues_d",
showmeans=True, meanprops={"markersize": 1, "markeredgecolor": "k"}, flierprops={"marker": "."})
ax.axhline(y=0, color="grey", linewidth=.5)
ax.set(ylabel=f"skill score", xlabel="", title="summary of all stations")
handles, _ = ax.get_legend_handles_labels()
ax.legend(handles, self._labels)
plt.tight_layout()
- self._save(plot_folder, model_setup)
-
- @staticmethod
- def _save(plot_folder, model_setup):
- """
- Standard save method to store plot locally. The name of this plot is dynamic. It includes the model setup like
- 'CNN' and can additionally be adjusted using an extra name tag.
- :param plot_folder: path to save the plot
- :param model_setup: architecture type to specify plot name
- """
- plot_name = os.path.join(plot_folder, f"skill_score_bootstrap_{model_setup}.pdf")
- logging.debug(f"... save plot to {plot_name}")
- plt.savefig(plot_name, dpi=500)
- plt.close('all')
-class PlotTimeSeries(RunEnvironment):
+@TimeTrackingWrapper
+class PlotTimeSeries:
def __init__(self, stations: List, data_path: str, name: str, window_lead_time: int = None, plot_folder: str = ".",
sampling="daily"):
- super().__init__()
self._data_path = data_path
self._data_name = name
self._stations = stations
@@ -665,3 +622,103 @@ class PlotTimeSeries(RunEnvironment):
plot_name = os.path.join(os.path.abspath(plot_folder), 'timeseries_plot.pdf')
logging.debug(f"... save plot to {plot_name}")
return matplotlib.backends.backend_pdf.PdfPages(plot_name)
+
+
+@TimeTrackingWrapper
+class PlotAvailability(AbstractPlotClass):
+
+ def __init__(self, generators: Dict[str, DataGenerator], plot_folder: str = ".", sampling="daily",
+ summary_name="data availability"):
+ # create standard Gantt plot for all stations (currently in single pdf file with single page)
+ super().__init__(plot_folder, "data_availability")
+ self.sampling = self._get_sampling(sampling)
+ plot_dict = self._prepare_data(generators)
+ lgd = self._plot(plot_dict)
+ self._save(bbox_extra_artists=(lgd, ), bbox_inches="tight")
+ # create summary Gantt plot (is data in at least one station available)
+ self.plot_name += "_summary"
+ plot_dict_summary = self._summarise_data(generators, summary_name)
+ lgd = self._plot(plot_dict_summary)
+ self._save(bbox_extra_artists=(lgd, ), bbox_inches="tight")
+ # combination of station and summary plot, last element is summary broken bar
+ self.plot_name = "data_availability_combined"
+ plot_dict_summary.update(plot_dict)
+ lgd = self._plot(plot_dict_summary)
+ self._save(bbox_extra_artists=(lgd, ), bbox_inches="tight")
+
+ @staticmethod
+ def _get_sampling(sampling):
+ if sampling == "daily":
+ return "D"
+ elif sampling == "hourly":
+ return "h"
+
+ def _prepare_data(self, generators: Dict[str, DataGenerator]):
+ plt_dict = {}
+ for subset, generator in generators.items():
+ stations = generator.stations
+ for station in stations:
+ station_data = generator.get_data_generator(station)
+ labels = station_data.get_transposed_label().resample(datetime=self.sampling, skipna=True).mean()
+ labels_bool = labels.sel(window=1).notnull()
+ group = (labels_bool != labels_bool.shift(datetime=1)).cumsum()
+ plot_data = pd.DataFrame({"avail": labels_bool.values, "group": group.values}, index=labels.datetime.values)
+ t = plot_data.groupby("group").apply(lambda x: (x["avail"].head(1)[0], x.index[0], x.shape[0]))
+ t2 = [i[1:] for i in t if i[0]]
+
+ if plt_dict.get(station) is None:
+ plt_dict[station] = {subset: t2}
+ else:
+ plt_dict[station].update({subset: t2})
+ return plt_dict
+
+ def _summarise_data(self, generators: Dict[str, DataGenerator], summary_name: str):
+ plt_dict = {}
+ for subset, generator in generators.items():
+ all_data = None
+ stations = generator.stations
+ for station in stations:
+ station_data = generator.get_data_generator(station)
+ labels = station_data.get_transposed_label().resample(datetime=self.sampling, skipna=True).mean()
+ labels_bool = labels.sel(window=1).notnull()
+ if all_data is None:
+ all_data = labels_bool
+ else:
+ tmp = all_data.combine_first(labels_bool) # expand dims to merged datetime coords
+ all_data = np.logical_or(tmp, labels_bool).combine_first(all_data) # apply logical on merge and fill missing with all_data
+
+ group = (all_data != all_data.shift(datetime=1)).cumsum()
+ plot_data = pd.DataFrame({"avail": all_data.values, "group": group.values}, index=all_data.datetime.values)
+ t = plot_data.groupby("group").apply(lambda x: (x["avail"].head(1)[0], x.index[0], x.shape[0]))
+ t2 = [i[1:] for i in t if i[0]]
+ if plt_dict.get(summary_name) is None:
+ plt_dict[summary_name] = {subset: t2}
+ else:
+ plt_dict[summary_name].update({subset: t2})
+ return plt_dict
+
+
+ def _plot(self, plt_dict):
+ # colors = {"train": "orange", "val": "blueishgreen", "test": "skyblue"} # color names
+ colors = {"train": "#e69f00", "val": "#009e73", "test": "#56b4e9"} # hex code
+ # colors = {"train": (230, 159, 0), "val": (0, 158, 115), "test": (86, 180, 233)} # in rgb but as abs values
+ pos = 0
+ height = 0.8 # should be <= 1
+ yticklabels = []
+ number_of_stations = len(plt_dict.keys())
+ fig, ax = plt.subplots(figsize=(10, number_of_stations/3))
+ for station, d in sorted(plt_dict.items(), reverse=True):
+ pos += 1
+ for subset, color in colors.items():
+ plt_data = d.get(subset)
+ if plt_data is None:
+ continue
+ ax.broken_barh(plt_data, (pos, height), color=color, edgecolor="white")
+ yticklabels.append(station)
+
+ ax.set_ylim([height, number_of_stations + 1])
+ ax.set_yticks(np.arange(len(plt_dict.keys()))+1+height/2)
+ ax.set_yticklabels(yticklabels)
+ handles = [mpatches.Patch(color=c, label=k) for k, c in colors.items()]
+ lgd = plt.legend(handles=handles, bbox_to_anchor=(0, 1, 1, 0.2), loc="lower center", ncol=len(handles))
+ return lgd
diff --git a/src/run_modules/experiment_setup.py b/src/run_modules/experiment_setup.py
index 56c22a81e48421438816855770b7477e84e3a8d8..150399cb2e4997a6b9adfb30dfa3ff89de73d4ac 100644
--- a/src/run_modules/experiment_setup.py
+++ b/src/run_modules/experiment_setup.py
@@ -20,6 +20,9 @@ DEFAULT_VAR_ALL_DICT = {'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'max
'v': 'average_values', 'no': 'dma8eu', 'no2': 'dma8eu', 'cloudcover': 'average_values',
'pblheight': 'maximum'}
DEFAULT_TRANSFORMATION = {"scope": "data", "method": "standardise", "mean": "estimate"}
+DEFAULT_PLOT_LIST = ["PlotMonthlySummary", "PlotStationMap", "PlotClimatologicalSkillScore", "PlotTimeSeries",
+ "PlotCompetitiveSkillScore", "PlotBootstrapSkillScore", "plot_conditional_quantiles",
+ "PlotAvailability"]
class ExperimentSetup(RunEnvironment):
@@ -34,7 +37,10 @@ class ExperimentSetup(RunEnvironment):
limit_nan_fill=None, train_start=None, train_end=None, val_start=None, val_end=None, test_start=None,
test_end=None, use_all_stations_on_all_data_sets=True, trainable=None, fraction_of_train=None,
experiment_path=None, plot_path=None, forecast_path=None, overwrite_local_data=None, sampling="daily",
- create_new_model=None, bootstrap_path=None, permute_data_on_training=None, transformation=None):
+ create_new_model=None, bootstrap_path=None, permute_data_on_training=False, transformation=None,
+ train_min_length=None, val_min_length=None, test_min_length=None, extreme_values=None,
+ extremes_on_right_tail_only=None, evaluate_bootstraps=True, plot_list=None, number_of_bootstraps=None,
+ create_new_bootstraps=None):
# create run framework
super().__init__()
@@ -42,14 +48,18 @@ class ExperimentSetup(RunEnvironment):
# experiment setup
self._set_param("data_path", helpers.prepare_host(sampling=sampling))
self._set_param("create_new_model", create_new_model, default=True)
- if self.data_store.get("create_new_model", "general"):
+ if self.data_store.get("create_new_model"):
trainable = True
- data_path = self.data_store.get("data_path", "general")
+ data_path = self.data_store.get("data_path")
bootstrap_path = helpers.set_bootstrap_path(bootstrap_path, data_path, sampling)
self._set_param("bootstrap_path", bootstrap_path)
self._set_param("trainable", trainable, default=True)
self._set_param("fraction_of_training", fraction_of_train, default=0.8)
- self._set_param("permute_data", permute_data_on_training, default=False, scope="general.train")
+ self._set_param("extreme_values", extreme_values, default=None, scope="train")
+ self._set_param("extremes_on_right_tail_only", extremes_on_right_tail_only, default=False, scope="train")
+ self._set_param("upsampling", extreme_values is not None, scope="train")
+ upsampling = self.data_store.get("upsampling", "train")
+ self._set_param("permute_data", max([permute_data_on_training, upsampling]), scope="train")
# set experiment name
exp_date = self._get_parser_args(parser_args).get("experiment_date")
@@ -57,32 +67,32 @@ class ExperimentSetup(RunEnvironment):
sampling=sampling)
self._set_param("experiment_name", exp_name)
self._set_param("experiment_path", exp_path)
- helpers.check_path_and_create(self.data_store.get("experiment_path", "general"))
+ helpers.check_path_and_create(self.data_store.get("experiment_path"))
# set plot path
default_plot_path = os.path.join(exp_path, "plots")
self._set_param("plot_path", plot_path, default=default_plot_path)
- helpers.check_path_and_create(self.data_store.get("plot_path", "general"))
+ helpers.check_path_and_create(self.data_store.get("plot_path"))
# set results path
default_forecast_path = os.path.join(exp_path, "forecasts")
self._set_param("forecast_path", forecast_path, default_forecast_path)
- helpers.check_path_and_create(self.data_store.get("forecast_path", "general"))
+ helpers.check_path_and_create(self.data_store.get("forecast_path"))
# setup for data
self._set_param("stations", stations, default=DEFAULT_STATIONS)
self._set_param("network", network, default="AIRBASE")
self._set_param("station_type", station_type, default=None)
self._set_param("statistics_per_var", statistics_per_var, default=DEFAULT_VAR_ALL_DICT)
- self._set_param("variables", variables, default=list(self.data_store.get("statistics_per_var", "general").keys()))
+ self._set_param("variables", variables, default=list(self.data_store.get("statistics_per_var").keys()))
self._compare_variables_and_statistics()
- self._set_param("start", start, default="1997-01-01", scope="general")
- self._set_param("end", end, default="2017-12-31", scope="general")
+ self._set_param("start", start, default="1997-01-01")
+ self._set_param("end", end, default="2017-12-31")
self._set_param("window_history_size", window_history_size, default=13)
- self._set_param("overwrite_local_data", overwrite_local_data, default=False, scope="general.preprocessing")
+ self._set_param("overwrite_local_data", overwrite_local_data, default=False, scope="preprocessing")
self._set_param("sampling", sampling)
self._set_param("transformation", transformation, default=DEFAULT_TRANSFORMATION)
- self._set_param("transformation", None, scope="general.preprocessing")
+ self._set_param("transformation", None, scope="preprocessing")
# target
self._set_param("target_var", target_var, default="o3")
@@ -97,24 +107,36 @@ class ExperimentSetup(RunEnvironment):
self._set_param("limit_nan_fill", limit_nan_fill, default=1)
# train set parameters
- self._set_param("start", train_start, default="1997-01-01", scope="general.train")
- self._set_param("end", train_end, default="2007-12-31", scope="general.train")
+ self._set_param("start", train_start, default="1997-01-01", scope="train")
+ self._set_param("end", train_end, default="2007-12-31", scope="train")
+ self._set_param("min_length", train_min_length, default=90, scope="train")
# validation set parameters
- self._set_param("start", val_start, default="2008-01-01", scope="general.val")
- self._set_param("end", val_end, default="2009-12-31", scope="general.val")
+ self._set_param("start", val_start, default="2008-01-01", scope="val")
+ self._set_param("end", val_end, default="2009-12-31", scope="val")
+ self._set_param("min_length", val_min_length, default=90, scope="val")
# test set parameters
- self._set_param("start", test_start, default="2010-01-01", scope="general.test")
- self._set_param("end", test_end, default="2017-12-31", scope="general.test")
+ self._set_param("start", test_start, default="2010-01-01", scope="test")
+ self._set_param("end", test_end, default="2017-12-31", scope="test")
+ self._set_param("min_length", test_min_length, default=90, scope="test")
# train_val set parameters
- self._set_param("start", self.data_store.get("start", "general.train"), scope="general.train_val")
- self._set_param("end", self.data_store.get("end", "general.val"), scope="general.train_val")
+ self._set_param("start", self.data_store.get("start", "train"), scope="train_val")
+ self._set_param("end", self.data_store.get("end", "val"), scope="train_val")
+ train_val_min_length = sum([self.data_store.get("min_length", s) for s in ["train", "val"]])
+ self._set_param("min_length", train_val_min_length, default=180, scope="train_val")
# use all stations on all data sets (train, val, test)
self._set_param("use_all_stations_on_all_data_sets", use_all_stations_on_all_data_sets, default=True)
+ # set post-processing instructions
+ self._set_param("evaluate_bootstraps", evaluate_bootstraps, scope="general.postprocessing")
+ create_new_bootstraps = max([self.data_store.get("trainable", "general"), create_new_bootstraps or False])
+ self._set_param("create_new_bootstraps", create_new_bootstraps, scope="general.postprocessing")
+ self._set_param("number_of_bootstraps", number_of_bootstraps, default=20, scope="general.postprocessing")
+ self._set_param("plot_list", plot_list, default=DEFAULT_PLOT_LIST, scope="general.postprocessing")
+
def _set_param(self, param: str, value: Any, default: Any = None, scope: str = "general") -> None:
if value is None and default is not None:
value = default
@@ -137,8 +159,8 @@ class ExperimentSetup(RunEnvironment):
def _compare_variables_and_statistics(self):
logging.debug("check if all variables are included in statistics_per_var")
- stat = self.data_store.get("statistics_per_var", "general")
- var = self.data_store.get("variables", "general")
+ stat = self.data_store.get("statistics_per_var")
+ var = self.data_store.get("variables")
if not set(var).issubset(stat.keys()):
missing = set(var).difference(stat.keys())
raise ValueError(f"Comparison of given variables and statistics_per_var show that not all requested "
@@ -146,9 +168,9 @@ class ExperimentSetup(RunEnvironment):
f"statistics for the variables: {missing}")
def _check_target_var(self):
- target_var = helpers.to_list(self.data_store.get("target_var", "general"))
- stat = self.data_store.get("statistics_per_var", "general")
- var = self.data_store.get("variables", "general")
+ target_var = helpers.to_list(self.data_store.get("target_var"))
+ stat = self.data_store.get("statistics_per_var")
+ var = self.data_store.get("variables")
if not set(target_var).issubset(stat.keys()):
raise ValueError(f"Could not find target variable {target_var} in statistics_per_var.")
unused_vars = set(stat.keys()).difference(set(var).union(target_var))
diff --git a/src/run_modules/model_setup.py b/src/run_modules/model_setup.py
index 32ca0d2e82af32d8164d80ac42731e10f431a458..c558b5fc76ff336dc6a792ec0239fa3b64eab466 100644
--- a/src/run_modules/model_setup.py
+++ b/src/run_modules/model_setup.py
@@ -10,8 +10,9 @@ import tensorflow as tf
from src.model_modules.keras_extensions import HistoryAdvanced, CallbackHandler
# from src.model_modules.model_class import MyBranchedModel as MyModel
-from src.model_modules.model_class import MyLittleModel as MyModel
-# from src.model_modules.model_class import MyTowerModel as MyModel
+# from src.model_modules.model_class import MyLittleModel as MyModel
+from src.model_modules.model_class import MyTowerModel as MyModel
+# from src.model_modules.model_class import MyPaperModel as MyModel
from src.run_modules.run_environment import RunEnvironment
@@ -22,15 +23,15 @@ class ModelSetup(RunEnvironment):
# create run framework
super().__init__()
self.model = None
- path = self.data_store.get("experiment_path", "general")
- exp_name = self.data_store.get("experiment_name", "general")
- self.scope = "general.model"
+ path = self.data_store.get("experiment_path")
+ exp_name = self.data_store.get("experiment_name")
+ self.scope = "model"
self.path = os.path.join(path, f"{exp_name}_%s")
self.model_name = self.path % "%s.h5"
self.checkpoint_name = self.path % "model-best.h5"
self.callbacks_name = self.path % "model-best-callbacks-%s.pickle"
- self._trainable = self.data_store.get("trainable", "general")
- self._create_new_model = self.data_store.get("create_new_model", "general")
+ self._trainable = self.data_store.get("trainable")
+ self._create_new_model = self.data_store.get("create_new_model")
self._run()
def _run(self):
@@ -55,7 +56,7 @@ class ModelSetup(RunEnvironment):
self.compile_model()
def _set_channels(self):
- channels = self.data_store.get("generator", "general.train")[0][0].shape[-1]
+ channels = self.data_store.get("generator", "train")[0][0].shape[-1]
self.data_store.set("channels", channels, self.scope)
def compile_model(self):
@@ -69,11 +70,12 @@ class ModelSetup(RunEnvironment):
Set all callbacks for the training phase. Add all callbacks with the .add_callback statement. Finally, the
advanced model checkpoint is added.
"""
- lr = self.data_store.get("lr_decay", scope="general.model")
+ lr = self.data_store.get_default("lr_decay", scope="model", default=None)
hist = HistoryAdvanced()
- self.data_store.set("hist", hist, scope="general.model")
+ self.data_store.set("hist", hist, scope="model")
callbacks = CallbackHandler()
- callbacks.add_callback(lr, self.callbacks_name % "lr", "lr")
+ if lr:
+ callbacks.add_callback(lr, self.callbacks_name % "lr", "lr")
callbacks.add_callback(hist, self.callbacks_name % "hist", "hist")
callbacks.create_model_checkpoint(filepath=self.checkpoint_name, verbose=1, monitor='val_loss',
save_best_only=True, mode='auto')
diff --git a/src/run_modules/post_processing.py b/src/run_modules/post_processing.py
index 0a61ee4f07d0c6eccf698aa16d3de9d7275e75f6..8a962888ec0b789a14a24b20c97148e7a8315b30 100644
--- a/src/run_modules/post_processing.py
+++ b/src/run_modules/post_processing.py
@@ -2,6 +2,7 @@ __author__ = "Lukas Leufen, Felix Kleinert"
__date__ = '2019-12-11'
+import inspect
import logging
import os
@@ -17,11 +18,14 @@ from src.data_handling.bootstraps import BootStraps
from src.datastore import NameNotFoundInDataStore
from src.helpers import TimeTracking
from src.model_modules.linear_model import OrdinaryLeastSquaredModel
+from src.model_modules.model_class import AbstractModelClass
from src.plotting.postprocessing_plotting import PlotMonthlySummary, PlotStationMap, PlotClimatologicalSkillScore, \
- PlotCompetitiveSkillScore, PlotTimeSeries, PlotBootstrapSkillScore
+ PlotCompetitiveSkillScore, PlotTimeSeries, PlotBootstrapSkillScore, PlotAvailability
from src.plotting.postprocessing_plotting import plot_conditional_quantiles
from src.run_modules.run_environment import RunEnvironment
+from typing import Dict
+
class PostProcessing(RunEnvironment):
@@ -29,14 +33,15 @@ class PostProcessing(RunEnvironment):
super().__init__()
self.model: keras.Model = self._load_model()
self.ols_model = None
- self.batch_size: int = self.data_store.get_default("batch_size", "general.model", 64)
- self.test_data: DataGenerator = self.data_store.get("generator", "general.test")
+ self.batch_size: int = self.data_store.get_default("batch_size", "model", 64)
+ self.test_data: DataGenerator = self.data_store.get("generator", "test")
self.test_data_distributed = Distributor(self.test_data, self.model, self.batch_size)
- self.train_data: DataGenerator = self.data_store.get("generator", "general.train")
- self.train_val_data: DataGenerator = self.data_store.get("generator", "general.train_val")
- self.plot_path: str = self.data_store.get("plot_path", "general")
- self.target_var = self.data_store.get("target_var", "general")
- self._sampling = self.data_store.get("sampling", "general")
+ self.train_data: DataGenerator = self.data_store.get("generator", "train")
+ self.val_data: DataGenerator = self.data_store.get("generator", "val")
+ self.train_val_data: DataGenerator = self.data_store.get("generator", "train_val")
+ self.plot_path: str = self.data_store.get("plot_path")
+ self.target_var = self.data_store.get("target_var")
+ self._sampling = self.data_store.get("sampling")
self.skill_scores = None
self.bootstrap_skill_scores = None
self._run()
@@ -50,93 +55,168 @@ class PostProcessing(RunEnvironment):
self.make_prediction()
logging.info("take a look on the next reported time measure. If this increases a lot, one should think to "
"skip make_prediction() whenever it is possible to save time.")
- self.bootstrap_skill_scores = self.create_boot_straps()
- self.skill_scores = self.calculate_skill_scores()
- self.plot()
- def create_boot_straps(self):
+ # bootstraps
+ if self.data_store.get("evaluate_bootstraps", "general.postprocessing"):
+ with TimeTracking(name="calculate bootstraps"):
+ create_new_bootstraps = self.data_store.get("create_new_bootstraps", "general.postprocessing")
+ self.bootstrap_postprocessing(create_new_bootstraps)
+
+ # skill scores
+ with TimeTracking(name="calculate skill scores"):
+ self.skill_scores = self.calculate_skill_scores()
+ # plotting
+ self.plot()
+
+ def bootstrap_postprocessing(self, create_new_bootstraps: bool, _iter: int = 0) -> None:
+ """
+ Create skill scores of bootstrapped data. Also creates these bootstraps if create_new_bootstraps is true or a
+ failure occurred during skill score calculation. Sets class attribute bootstrap_skill_scores.
+ :param create_new_bootstraps: calculate all bootstrap predictions and overwrite already available predictions
+ :param _iter: internal counter to reduce unnecessary recursive calls (maximum number is 2, otherwise something
+ went wrong).
+ """
+ try:
+ if create_new_bootstraps:
+ self.create_bootstrap_forecast()
+ self.bootstrap_skill_scores = self.calculate_bootstrap_skill_scores()
+ except FileNotFoundError:
+ if _iter != 0:
+ raise RuntimeError("bootstrap_postprocessing is called for the 2nd time. This means, that calling"
+ "manually the reason for the failure.")
+ logging.info("Couldn't load all files, restart bootstrap postprocessing with create_new_bootstraps=True.")
+ self.bootstrap_postprocessing(True, _iter=1)
+
+ def create_bootstrap_forecast(self) -> None:
+ """
+ Creates the bootstrapped predictions for all stations and variables. These forecasts are saved in bootstrap_path
+ with the names `bootstraps_{var}_{station}.nc` and `bootstraps_labels_{station}.nc`.
+ """
# forecast
+ with TimeTracking(name=inspect.stack()[0].function):
+ # extract all requirements from data store
+ bootstrap_path = self.data_store.get("bootstrap_path")
+ forecast_path = self.data_store.get("forecast_path")
+ number_of_bootstraps = self.data_store.get("number_of_bootstraps", "general.postprocessing")
+
+ # set bootstrap class
+ bootstraps = BootStraps(self.test_data, bootstrap_path, number_of_bootstraps)
+
+ # create bootstrapped predictions for all stations and variables and save it to disk
+ dims = ["index", "ahead", "type"]
+ for station in bootstraps.stations:
+ with TimeTracking(name=station):
+ logging.info(station)
+ for var in bootstraps.variables:
+ station_bootstrap = bootstraps.get_generator(station, var)
+
+ # make bootstrap predictions
+ bootstrap_predictions = self.model.predict_generator(generator=station_bootstrap,
+ workers=2,
+ use_multiprocessing=True)
+ if isinstance(bootstrap_predictions, list): # if model is branched model
+ bootstrap_predictions = bootstrap_predictions[-1]
+ # save bootstrap predictions separately for each station and variable combination
+ bootstrap_predictions = np.expand_dims(bootstrap_predictions, axis=-1)
+ shape = bootstrap_predictions.shape
+ coords = (range(shape[0]), range(1, shape[1] + 1))
+ tmp = xr.DataArray(bootstrap_predictions, coords=(*coords, [var]), dims=dims)
+ file_name = os.path.join(forecast_path, f"bootstraps_{var}_{station}.nc")
+ tmp.to_netcdf(file_name)
+ # store also true labels for each station
+ labels = np.expand_dims(bootstraps.get_labels(station), axis=-1)
+ file_name = os.path.join(forecast_path, f"bootstraps_labels_{station}.nc")
+ labels = xr.DataArray(labels, coords=(*coords, ["obs"]), dims=dims)
+ labels.to_netcdf(file_name)
+
+ def calculate_bootstrap_skill_scores(self) -> Dict[str, xr.DataArray]:
+ """
+ Use already created bootstrap predictions and the original predictions (the not-bootstrapped ones) and calculate
+ skill scores for the bootstraps. The result is saved as a xarray DataArray in a dictionary structure separated
+ for each station (keys of dictionary).
+ :return: The result dictionary with station-wise skill scores
+ """
- bootstrap_path = self.data_store.get("bootstrap_path", "general")
- forecast_path = self.data_store.get("forecast_path", "general")
- window_lead_time = self.data_store.get("window_lead_time", "general")
- bootstraps = BootStraps(self.test_data, bootstrap_path, 20)
- with TimeTracking(name="boot predictions"):
- bootstrap_predictions = self.model.predict_generator(generator=bootstraps.boot_strap_generator(),
- steps=bootstraps.get_boot_strap_generator_length())
- if isinstance(bootstrap_predictions, list):
- bootstrap_predictions = bootstrap_predictions[-1]
- bootstrap_meta = np.array(bootstraps.get_boot_strap_meta())
- variables = np.unique(bootstrap_meta[:, 0])
- for station in np.unique(bootstrap_meta[:, 1]):
- coords = None
- for boot in variables:
- ind = np.all(bootstrap_meta == [boot, station], axis=1)
- length = sum(ind)
- sel = bootstrap_predictions[ind].reshape((length, window_lead_time, 1))
- coords = (range(length), range(1, window_lead_time + 1))
- tmp = xr.DataArray(sel, coords=(*coords, [boot]), dims=["index", "ahead", "type"])
- file_name = os.path.join(forecast_path, f"bootstraps_{boot}_{station}.nc")
- tmp.to_netcdf(file_name)
- labels = bootstraps.get_labels(station).reshape((length, window_lead_time, 1))
- file_name = os.path.join(forecast_path, f"bootstraps_labels_{station}.nc")
- labels = xr.DataArray(labels, coords=(*coords, ["obs"]), dims=["index", "ahead", "type"])
- labels.to_netcdf(file_name)
-
- # file_name = os.path.join(forecast_path, f"bootstraps_orig.nc")
- # orig = xr.open_dataarray(file_name)
-
-
- # calc skill scores
- skill_scores = statistics.SkillScores(None)
- score = {}
- for station in np.unique(bootstrap_meta[:, 1]):
- file_name = os.path.join(forecast_path, f"bootstraps_labels_{station}.nc")
- labels = xr.open_dataarray(file_name)
- shape = labels.shape
- orig = bootstraps.get_orig_prediction(forecast_path, f"forecasts_norm_{station}_test.nc").reshape(shape)
- orig = xr.DataArray(orig, coords=(range(shape[0]), range(1, shape[1] + 1), ["orig"]), dims=["index", "ahead", "type"])
- skill = pd.DataFrame(columns=range(1, window_lead_time + 1))
- for boot in variables:
- file_name = os.path.join(forecast_path, f"bootstraps_{boot}_{station}.nc")
- boot_data = xr.open_dataarray(file_name)
- boot_data = boot_data.combine_first(labels)
- boot_data = boot_data.combine_first(orig)
- boot_scores = []
- for iahead in range(window_lead_time):
- data = boot_data.sel(ahead=iahead + 1)
- boot_scores.append(skill_scores.general_skill_score(data, forecast_name=boot, reference_name="orig"))
- skill.loc[boot] = np.array(boot_scores)
- score[station] = xr.DataArray(skill, dims=["boot_var", "ahead"])
- return score
+ with TimeTracking(name=inspect.stack()[0].function):
+ # extract all requirements from data store
+ bootstrap_path = self.data_store.get("bootstrap_path")
+ forecast_path = self.data_store.get("forecast_path")
+ window_lead_time = self.data_store.get("window_lead_time")
+ number_of_bootstraps = self.data_store.get("number_of_bootstraps", "postprocessing")
+ bootstraps = BootStraps(self.test_data, bootstrap_path, number_of_bootstraps)
+
+ skill_scores = statistics.SkillScores(None)
+ score = {}
+ for station in self.test_data.stations:
+ logging.info(station)
+
+ # get station labels
+ file_name = os.path.join(forecast_path, f"bootstraps_labels_{station}.nc")
+ labels = xr.open_dataarray(file_name)
+ shape = labels.shape
+
+ # get original forecasts
+ orig = bootstraps.get_orig_prediction(forecast_path, f"forecasts_norm_{station}_test.nc").reshape(shape)
+ coords = (range(shape[0]), range(1, shape[1] + 1), ["orig"])
+ orig = xr.DataArray(orig, coords=coords, dims=["index", "ahead", "type"])
+
+ # calculate skill scores for each variable
+ skill = pd.DataFrame(columns=range(1, window_lead_time + 1))
+ for boot in self.test_data.variables:
+ file_name = os.path.join(forecast_path, f"bootstraps_{boot}_{station}.nc")
+ boot_data = xr.open_dataarray(file_name)
+ boot_data = boot_data.combine_first(labels).combine_first(orig)
+ boot_scores = []
+ for ahead in range(1, window_lead_time + 1):
+ data = boot_data.sel(ahead=ahead)
+ boot_scores.append(skill_scores.general_skill_score(data, forecast_name=boot, reference_name="orig"))
+ skill.loc[boot] = np.array(boot_scores)
+
+ # collect all results in single dictionary
+ score[station] = xr.DataArray(skill, dims=["boot_var", "ahead"])
+ return score
def _load_model(self):
try:
- model = self.data_store.get("best_model", "general")
+ model = self.data_store.get("best_model")
except NameNotFoundInDataStore:
logging.info("no model saved in data store. trying to load model from experiment path")
- model_name = self.data_store.get("model_name", "general.model")
- model = keras.models.load_model(model_name)
+ model_name = self.data_store.get("model_name", "model")
+ model_class: AbstractModelClass = self.data_store.get("model", "model")
+ model = keras.models.load_model(model_name, custom_objects=model_class.custom_objects)
return model
def plot(self):
logging.debug("Run plotting routines...")
- path = self.data_store.get("forecast_path", "general")
-
- plot_conditional_quantiles(self.test_data.stations, pred_name="CNN", ref_name="obs",
- forecast_path=path, plot_name_affix="cali-ref", plot_folder=self.plot_path)
- plot_conditional_quantiles(self.test_data.stations, pred_name="obs", ref_name="CNN",
- forecast_path=path, plot_name_affix="like-bas", plot_folder=self.plot_path)
- PlotStationMap(generators={'b': self.test_data}, plot_folder=self.plot_path)
- PlotMonthlySummary(self.test_data.stations, path, r"forecasts_%s_test.nc", self.target_var,
- plot_folder=self.plot_path)
- PlotClimatologicalSkillScore(self.skill_scores[1], plot_folder=self.plot_path, model_setup="CNN")
- PlotClimatologicalSkillScore(self.skill_scores[1], plot_folder=self.plot_path, score_only=False,
- extra_name_tag="all_terms_", model_setup="CNN")
- PlotCompetitiveSkillScore(self.skill_scores[0], plot_folder=self.plot_path, model_setup="CNN")
- PlotBootstrapSkillScore(self.bootstrap_skill_scores, plot_folder=self.plot_path, model_setup="CNN")
- PlotTimeSeries(self.test_data.stations, path, r"forecasts_%s_test.nc", plot_folder=self.plot_path, sampling=self._sampling)
+ path = self.data_store.get("forecast_path")
+
+ plot_list = self.data_store.get("plot_list", "postprocessing")
+
+ if self.bootstrap_skill_scores is not None and "PlotBootstrapSkillScore" in plot_list:
+ PlotBootstrapSkillScore(self.bootstrap_skill_scores, plot_folder=self.plot_path, model_setup="CNN")
+ if "plot_conditional_quantiles" in plot_list:
+ plot_conditional_quantiles(self.test_data.stations, pred_name="CNN", ref_name="obs",
+ forecast_path=path, plot_name_affix="cali-ref", plot_folder=self.plot_path)
+ plot_conditional_quantiles(self.test_data.stations, pred_name="obs", ref_name="CNN",
+ forecast_path=path, plot_name_affix="like-bas", plot_folder=self.plot_path)
+ if "PlotStationMap" in plot_list:
+ PlotStationMap(generators={'b': self.test_data}, plot_folder=self.plot_path)
+ if "PlotMonthlySummary" in plot_list:
+ PlotMonthlySummary(self.test_data.stations, path, r"forecasts_%s_test.nc", self.target_var,
+ plot_folder=self.plot_path)
+ if "PlotClimatologicalSkillScore" in plot_list:
+ PlotClimatologicalSkillScore(self.skill_scores[1], plot_folder=self.plot_path, model_setup="CNN")
+ PlotClimatologicalSkillScore(self.skill_scores[1], plot_folder=self.plot_path, score_only=False,
+ extra_name_tag="all_terms_", model_setup="CNN")
+ if "PlotCompetitiveSkillScore" in plot_list:
+ PlotCompetitiveSkillScore(self.skill_scores[0], plot_folder=self.plot_path, model_setup="CNN")
+ if "PlotTimeSeries" in plot_list:
+ PlotTimeSeries(self.test_data.stations, path, r"forecasts_%s_test.nc", plot_folder=self.plot_path,
+ sampling=self._sampling)
+ if "PlotAvailability" in plot_list:
+ avail_data = {"train": self.train_data, "val": self.val_data, "test": self.test_data}
+ PlotAvailability(avail_data, plot_folder=self.plot_path)
def calculate_test_score(self):
test_score = self.model.evaluate_generator(generator=self.test_data_distributed.distribute_on_batches(),
@@ -145,7 +225,7 @@ class PostProcessing(RunEnvironment):
self._save_test_score(test_score)
def _save_test_score(self, score):
- path = self.data_store.get("experiment_path", "general")
+ path = self.data_store.get("experiment_path")
with open(os.path.join(path, "test_scores.txt")) as f:
for index, item in enumerate(score):
f.write(f"{self.model.metrics[index]}, {item}\n")
@@ -188,7 +268,7 @@ class PostProcessing(RunEnvironment):
OLS=ols_prediction)
# save all forecasts locally
- path = self.data_store.get("forecast_path", "general")
+ path = self.data_store.get("forecast_path")
prefix = "forecasts_norm" if normalised else "forecasts"
file = os.path.join(path, f"{prefix}_{data.station[0]}_test.nc")
all_predictions.to_netcdf(file)
@@ -216,7 +296,7 @@ class PostProcessing(RunEnvironment):
tmp_persi = data.observation.copy().sel({'window': 0})
if not normalised:
tmp_persi = statistics.apply_inverse_transformation(tmp_persi, mean, std, transformation_method)
- window_lead_time = self.data_store.get("window_lead_time", "general")
+ window_lead_time = self.data_store.get("window_lead_time")
persistence_prediction.values = np.expand_dims(np.tile(tmp_persi.squeeze('Stations'), (window_lead_time, 1)),
axis=1)
return persistence_prediction
@@ -305,8 +385,8 @@ class PostProcessing(RunEnvironment):
return None
def calculate_skill_scores(self):
- path = self.data_store.get("forecast_path", "general")
- window_lead_time = self.data_store.get("window_lead_time", "general")
+ path = self.data_store.get("forecast_path")
+ window_lead_time = self.data_store.get("window_lead_time")
skill_score_competitive = {}
skill_score_climatological = {}
for station in self.test_data.stations:
diff --git a/src/run_modules/pre_processing.py b/src/run_modules/pre_processing.py
index 1d014c9e6f4fc0a9168c4d3d31b1141c39fff2a1..551ea599a3114b7b97f5bcb146cf6e131e324eb5 100644
--- a/src/run_modules/pre_processing.py
+++ b/src/run_modules/pre_processing.py
@@ -3,16 +3,21 @@ __date__ = '2019-11-25'
import logging
+import os
from typing import Tuple, Dict, List
+import numpy as np
+import pandas as pd
+
from src.data_handling.data_generator import DataGenerator
-from src.helpers import TimeTracking
+from src.helpers import TimeTracking, check_path_and_create
from src.join import EmptyQueryResult
from src.run_modules.run_environment import RunEnvironment
DEFAULT_ARGS_LIST = ["data_path", "network", "stations", "variables", "interpolate_dim", "target_dim", "target_var"]
-DEFAULT_KWARGS_LIST = ["limit_nan_fill", "window_history_size", "window_lead_time", "statistics_per_var",
- "station_type", "overwrite_local_data", "start", "end", "sampling", "transformation"]
+DEFAULT_KWARGS_LIST = ["limit_nan_fill", "window_history_size", "window_lead_time", "statistics_per_var", "min_length",
+ "station_type", "overwrite_local_data", "start", "end", "sampling", "transformation",
+ "extreme_values", "extremes_on_right_tail_only"]
class PreProcessing(RunEnvironment):
@@ -33,26 +38,78 @@ class PreProcessing(RunEnvironment):
self._run()
def _run(self):
- args = self.data_store.create_args_dict(DEFAULT_ARGS_LIST, scope="general.preprocessing")
- kwargs = self.data_store.create_args_dict(DEFAULT_KWARGS_LIST, scope="general.preprocessing")
- stations = self.data_store.get("stations", "general")
- valid_stations = self.check_valid_stations(args, kwargs, stations, load_tmp=False, save_tmp=False)
- self.data_store.set("stations", valid_stations, "general")
+ args = self.data_store.create_args_dict(DEFAULT_ARGS_LIST, scope="preprocessing")
+ kwargs = self.data_store.create_args_dict(DEFAULT_KWARGS_LIST, scope="preprocessing")
+ stations = self.data_store.get("stations")
+ valid_stations = self.check_valid_stations(args, kwargs, stations, load_tmp=False, save_tmp=False, name="all")
+ self.data_store.set("stations", valid_stations)
self.split_train_val_test()
self.report_pre_processing()
def report_pre_processing(self):
logging.debug(20 * '##')
- n_train = len(self.data_store.get('generator', 'general.train'))
- n_val = len(self.data_store.get('generator', 'general.val'))
- n_test = len(self.data_store.get('generator', 'general.test'))
+ n_train = len(self.data_store.get('generator', 'train'))
+ n_val = len(self.data_store.get('generator', 'val'))
+ n_test = len(self.data_store.get('generator', 'test'))
n_total = n_train + n_val + n_test
logging.debug(f"Number of all stations: {n_total}")
logging.debug(f"Number of training stations: {n_train}")
logging.debug(f"Number of val stations: {n_val}")
logging.debug(f"Number of test stations: {n_test}")
- logging.debug(f"TEST SHAPE OF GENERATOR CALL: {self.data_store.get('generator', 'general.test')[0][0].shape}"
- f"{self.data_store.get('generator', 'general.test')[0][1].shape}")
+ logging.debug(f"TEST SHAPE OF GENERATOR CALL: {self.data_store.get('generator', 'test')[0][0].shape}"
+ f"{self.data_store.get('generator', 'test')[0][1].shape}")
+ self.create_latex_report()
+
+ def create_latex_report(self):
+ """
+ This function creates tables with information on the station meta data and a summary on subset sample sizes.
+
+ * station_sample_size.md: see table below
+ * station_sample_size.tex: same as table below, but as latex table
+ * station_sample_size_short.tex: reduced size table without any meta data besides station ID, as latex table
+
+ All tables are stored inside experiment_path inside the folder latex_report. The table format (e.g. which meta
+ data is highlighted) is currently hardcoded to have a stable table style. If further styles are needed, it is
+ better to add an additional style than modifying the existing table styles.
+
+ | stat. ID | station_name | station_lon | station_lat | station_alt | train | val | test |
+ |------------|-------------------------------------------|---------------|---------------|---------------|---------|-------|--------|
+ | DEBW013 | Stuttgart Bad Cannstatt | 9.2297 | 48.8088 | 235 | 1434 | 712 | 1080 |
+ | DEBW076 | Baden-Baden | 8.2202 | 48.7731 | 148 | 3037 | 722 | 710 |
+ | DEBW087 | Schwäbische_Alb | 9.2076 | 48.3458 | 798 | 3044 | 714 | 1087 |
+ | DEBW107 | Tübingen | 9.0512 | 48.5077 | 325 | 1803 | 715 | 1087 |
+ | DEBY081 | Garmisch-Partenkirchen/Kreuzeckbahnstraße | 11.0631 | 47.4764 | 735 | 2935 | 525 | 714 |
+ | # Stations | nan | nan | nan | nan | 6 | 6 | 6 |
+ | # Samples | nan | nan | nan | nan | 12253 | 3388 | 4678 |
+
+ """
+ meta_data = ['station_name', 'station_lon', 'station_lat', 'station_alt']
+ meta_round = ["station_lon", "station_lat", "station_alt"]
+ precision = 4
+ path = os.path.join(self.data_store.get("experiment_path"), "latex_report")
+ check_path_and_create(path)
+ set_names = ["train", "val", "test"]
+ df = pd.DataFrame(columns=meta_data+set_names)
+ for set_name in set_names:
+ data: DataGenerator = self.data_store.get("generator", set_name)
+ for station in data.stations:
+ df.loc[station, set_name] = data.get_data_generator(station).get_transposed_label().shape[0]
+ if df.loc[station, meta_data].isnull().any():
+ df.loc[station, meta_data] = data.get_data_generator(station).meta.loc[meta_data].values.flatten()
+ df.loc["# Samples", set_name] = df.loc[:, set_name].sum()
+ df.loc["# Stations", set_name] = df.loc[:, set_name].count()
+ df[meta_round] = df[meta_round].astype(float).round(precision)
+ df.sort_index(inplace=True)
+ df = df.reindex(df.index.drop(["# Stations", "# Samples"]).to_list() + ["# Stations", "# Samples"], )
+ df.index.name = 'stat. ID'
+ column_format = np.repeat('c', df.shape[1]+1)
+ column_format[0] = 'l'
+ column_format[-1] = 'r'
+ column_format = ''.join(column_format.tolist())
+ df.to_latex(os.path.join(path, "station_sample_size.tex"), na_rep='---', column_format=column_format)
+ df.to_markdown(open(os.path.join(path, "station_sample_size.md"), mode="w", encoding='utf-8'), tablefmt="github")
+ df.drop(meta_data, axis=1).to_latex(os.path.join(path, "station_sample_size_short.tex"), na_rep='---',
+ column_format=column_format)
def split_train_val_test(self) -> None:
"""
@@ -60,8 +117,8 @@ class PreProcessing(RunEnvironment):
but as an separate generator). IMPORTANT: Do not change to order of the execution of create_set_split. The train
subset needs always to be executed at first, to set a proper transformation.
"""
- fraction_of_training = self.data_store.get("fraction_of_training", "general")
- stations = self.data_store.get("stations", "general")
+ fraction_of_training = self.data_store.get("fraction_of_training")
+ stations = self.data_store.get("stations")
train_index, val_index, test_index, train_val_index = self.split_set_indices(len(stations), fraction_of_training)
subset_names = ["train", "val", "test", "train_val"]
if subset_names[0] != "train": # pragma: no cover
@@ -96,27 +153,26 @@ class PreProcessing(RunEnvironment):
sure, that the train set is executed first, and all other subsets afterwards.
:param index_list: list of all stations to use for the set. If attribute use_all_stations_on_all_data_sets=True,
this list is ignored.
- :param set_name: name to load/save all information from/to data store without the leading general prefix.
+ :param set_name: name to load/save all information from/to data store.
"""
- scope = f"general.{set_name}"
- args = self.data_store.create_args_dict(DEFAULT_ARGS_LIST, scope)
- kwargs = self.data_store.create_args_dict(DEFAULT_KWARGS_LIST, scope)
+ args = self.data_store.create_args_dict(DEFAULT_ARGS_LIST, scope=set_name)
+ kwargs = self.data_store.create_args_dict(DEFAULT_KWARGS_LIST, scope=set_name)
stations = args["stations"]
- if self.data_store.get("use_all_stations_on_all_data_sets", scope):
+ if self.data_store.get("use_all_stations_on_all_data_sets", scope=set_name):
set_stations = stations
else:
set_stations = stations[index_list]
logging.debug(f"{set_name.capitalize()} stations (len={len(set_stations)}): {set_stations}")
- set_stations = self.check_valid_stations(args, kwargs, set_stations, load_tmp=False)
- self.data_store.set("stations", set_stations, scope)
- set_args = self.data_store.create_args_dict(DEFAULT_ARGS_LIST, scope)
+ set_stations = self.check_valid_stations(args, kwargs, set_stations, load_tmp=False, name=set_name)
+ self.data_store.set("stations", set_stations, scope=set_name)
+ set_args = self.data_store.create_args_dict(DEFAULT_ARGS_LIST, scope=set_name)
data_set = DataGenerator(**set_args, **kwargs)
- self.data_store.set("generator", data_set, scope)
+ self.data_store.set("generator", data_set, scope=set_name)
if set_name == "train":
- self.data_store.set("transformation", data_set.transformation, "general")
+ self.data_store.set("transformation", data_set.transformation)
@staticmethod
- def check_valid_stations(args: Dict, kwargs: Dict, all_stations: List[str], load_tmp=True, save_tmp=True):
+ def check_valid_stations(args: Dict, kwargs: Dict, all_stations: List[str], load_tmp=True, save_tmp=True, name=None):
"""
Check if all given stations in `all_stations` are valid. Valid means, that there is data available for the given
time range (is included in `kwargs`). The shape and the loading time are logged in debug mode.
@@ -125,17 +181,19 @@ class PreProcessing(RunEnvironment):
:param kwargs: positional parameters for the DataGenerator class (e.g. `start`, `interpolate_method`,
`window_lead_time`).
:param all_stations: All stations to check.
+ :param name: name to display in the logging info message
:return: Corrected list containing only valid station IDs.
"""
t_outer = TimeTracking()
t_inner = TimeTracking(start=False)
- logging.info("check valid stations started")
+ logging.info(f"check valid stations started{' (%s)' % name if name else ''}")
valid_stations = []
# all required arguments of the DataGenerator can be found in args, positional arguments in args and kwargs
data_gen = DataGenerator(**args, **kwargs)
- for station in all_stations:
+ for pos, station in enumerate(all_stations):
t_inner.run()
+ logging.info(f"check station {station} ({pos + 1} / {len(all_stations)})")
try:
data = data_gen.get_data_generator(key=station, load_local_tmp_storage=load_tmp,
save_local_tmp_storage=save_tmp)
diff --git a/src/run_modules/run_environment.py b/src/run_modules/run_environment.py
index 56c017290eea4d11881b9b131378d8c5995f0b29..7bd5027788934322d704192e1dff2995539fe245 100644
--- a/src/run_modules/run_environment.py
+++ b/src/run_modules/run_environment.py
@@ -2,9 +2,13 @@ __author__ = "Lukas Leufen"
__date__ = '2019-11-25'
import logging
+import os
+import shutil
import time
+from src.helpers import Logger
from src.datastore import DataStoreByScope as DataStoreObject
+from src.datastore import NameNotFoundInDataStore
from src.helpers import TimeTracking
@@ -16,6 +20,7 @@ class RunEnvironment(object):
del_by_exit = False
data_store = DataStoreObject()
+ logger = Logger()
def __init__(self):
"""
@@ -34,14 +39,30 @@ class RunEnvironment(object):
logging.info(f"{self.__class__.__name__} finished after {self.time}")
self.del_by_exit = True
if self.__class__.__name__ == "RunEnvironment":
+ self.__copy_log_file()
self.data_store.clear_data_store()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
+ if exc_type:
+ logging.error(exc_val, exc_info=(exc_type, exc_val, exc_tb))
self.__del__()
+ def __copy_log_file(self):
+ try:
+ counter = 0
+ filename_pattern = os.path.join(self.data_store.get("experiment_path"), "logging_%03i.log")
+ new_file = filename_pattern % counter
+ while os.path.exists(new_file):
+ counter += 1
+ new_file = filename_pattern % counter
+ logging.info(f"Copy log file to {new_file}")
+ shutil.copyfile(self.logger.log_file, new_file)
+ except (NameNotFoundInDataStore, FileNotFoundError):
+ pass
+
@staticmethod
def do_stuff(length=2):
time.sleep(length)
diff --git a/src/run_modules/training.py b/src/run_modules/training.py
index df60c4f2f8dff4a9acb82920ad3c1d203813033d..2d949af8c68f244c0a0da2bad6580c616695da8d 100644
--- a/src/run_modules/training.py
+++ b/src/run_modules/training.py
@@ -9,25 +9,27 @@ import pickle
import keras
from src.data_handling.data_distributor import Distributor
-from src.model_modules.keras_extensions import LearningRateDecay, ModelCheckpointAdvanced, CallbackHandler
+from src.model_modules.keras_extensions import LearningRateDecay, CallbackHandler
from src.plotting.training_monitoring import PlotModelHistory, PlotModelLearningRate
from src.run_modules.run_environment import RunEnvironment
+from typing import Union
+
class Training(RunEnvironment):
def __init__(self):
super().__init__()
- self.model: keras.Model = self.data_store.get("model", "general.model")
- self.train_set = None
- self.val_set = None
- self.test_set = None
- self.batch_size = self.data_store.get("batch_size", "general.model")
- self.epochs = self.data_store.get("epochs", "general.model")
- self.callbacks: CallbackHandler = self.data_store.get("callbacks", "general.model")
- self.experiment_name = self.data_store.get("experiment_name", "general")
- self._trainable = self.data_store.get("trainable", "general")
- self._create_new_model = self.data_store.get("create_new_model", "general")
+ self.model: keras.Model = self.data_store.get("model", "model")
+ self.train_set: Union[Distributor, None] = None
+ self.val_set: Union[Distributor, None] = None
+ self.test_set: Union[Distributor, None] = None
+ self.batch_size = self.data_store.get("batch_size", "model")
+ self.epochs = self.data_store.get("epochs", "model")
+ self.callbacks: CallbackHandler = self.data_store.get("callbacks", "model")
+ self.experiment_name = self.data_store.get("experiment_name")
+ self._trainable = self.data_store.get("trainable")
+ self._create_new_model = self.data_store.get("create_new_model")
self._run()
def _run(self) -> None:
@@ -64,9 +66,10 @@ class Training(RunEnvironment):
Set and distribute the generators for given mode regarding batch size
:param mode: name of set, should be from ["train", "val", "test"]
"""
- gen = self.data_store.get("generator", f"general.{mode}")
- permute_data = self.data_store.get_default("permute_data", f"general.{mode}", default=False)
- setattr(self, f"{mode}_set", Distributor(gen, self.model, self.batch_size, permute_data=permute_data))
+ gen = self.data_store.get("generator", mode)
+ # permute_data = self.data_store.get_default("permute_data", mode, default=False)
+ kwargs = self.data_store.create_args_dict(["permute_data", "upsampling"], scope=mode)
+ setattr(self, f"{mode}_set", Distributor(gen, self.model, self.batch_size, **kwargs))
def set_generators(self) -> None:
"""
@@ -86,6 +89,9 @@ class Training(RunEnvironment):
locally stored information and the corresponding model and proceed with the already started training.
"""
logging.info(f"Train with {len(self.train_set)} mini batches.")
+ logging.info(f"Train with option upsampling={self.train_set.upsampling}.")
+ logging.info(f"Train with option data_permutation={self.train_set.do_data_permutation}.")
+
checkpoint = self.callbacks.get_checkpoint()
if not os.path.exists(checkpoint.filepath) or self._create_new_model:
history = self.model.fit_generator(generator=self.train_set.distribute_on_batches(),
@@ -111,7 +117,10 @@ class Training(RunEnvironment):
callbacks=self.callbacks.get_callbacks(as_dict=False),
initial_epoch=initial_epoch)
history = hist
- lr = self.callbacks.get_callback_by_name("lr")
+ try:
+ lr = self.callbacks.get_callback_by_name("lr")
+ except IndexError:
+ lr = None
self.save_callbacks_as_json(history, lr)
self.load_best_model(checkpoint.filepath)
self.create_monitoring_plots(history, lr)
@@ -120,10 +129,10 @@ class Training(RunEnvironment):
"""
save model in local experiment directory. Model is named as <experiment_name>_<custom_model_name>.h5 .
"""
- model_name = self.data_store.get("model_name", "general.model")
+ model_name = self.data_store.get("model_name", "model")
logging.debug(f"save best model to {model_name}")
self.model.save(model_name)
- self.data_store.set("best_model", self.model, "general")
+ self.data_store.set("best_model", self.model)
def load_best_model(self, name: str) -> None:
"""
@@ -145,11 +154,12 @@ class Training(RunEnvironment):
:param history: history object of training
"""
logging.debug("saving callbacks")
- path = self.data_store.get("experiment_path", "general")
+ path = self.data_store.get("experiment_path")
with open(os.path.join(path, "history.json"), "w") as f:
json.dump(history.history, f)
- with open(os.path.join(path, "history_lr.json"), "w") as f:
- json.dump(lr_sc.lr, f)
+ if lr_sc:
+ with open(os.path.join(path, "history_lr.json"), "w") as f:
+ json.dump(lr_sc.lr, f)
def create_monitoring_plots(self, history: keras.callbacks.History, lr_sc: LearningRateDecay) -> None:
"""
@@ -159,8 +169,8 @@ class Training(RunEnvironment):
:param history: keras history object with losses to plot (must include 'loss' and 'val_loss')
:param lr_sc: learning rate decay object with 'lr' attribute
"""
- path = self.data_store.get("plot_path", "general")
- name = self.data_store.get("experiment_name", "general")
+ path = self.data_store.get("plot_path")
+ name = self.data_store.get("experiment_name")
# plot history of loss and mse (if available)
filename = os.path.join(path, f"{name}_history_loss.pdf")
@@ -174,4 +184,5 @@ class Training(RunEnvironment):
PlotModelHistory(filename=filename, history=history, plot_metric="mse", main_branch=multiple_branches_used)
# plot learning rate
- PlotModelLearningRate(filename=os.path.join(path, f"{name}_history_learning_rate.pdf"), lr_sc=lr_sc)
+ if lr_sc:
+ PlotModelLearningRate(filename=os.path.join(path, f"{name}_history_learning_rate.pdf"), lr_sc=lr_sc)
diff --git a/src/statistics.py b/src/statistics.py
index 26b2be8854c51584f20b753717ea94cc12967369..6510097fc3c31645bc0fa053a5ade05c3e4d908d 100644
--- a/src/statistics.py
+++ b/src/statistics.py
@@ -103,10 +103,9 @@ def mean_squared_error(a, b):
return np.square(a - b).mean()
-class SkillScores(RunEnvironment):
+class SkillScores:
def __init__(self, internal_data):
- super().__init__()
self.internal_data = internal_data
def skill_scores(self, window_lead_time):
diff --git a/test/test_data_handling/test_bootstraps.py b/test/test_data_handling/test_bootstraps.py
index 9dd23893ef903bfbd0595a482dceb32724c3b437..c2b814b7bf173b61b4967c83611cdd3de08ed91b 100644
--- a/test/test_data_handling/test_bootstraps.py
+++ b/test/test_data_handling/test_bootstraps.py
@@ -1,64 +1,293 @@
-from src.data_handling.bootstraps import BootStraps
+from src.data_handling.bootstraps import BootStraps, CreateShuffledData, BootStrapGenerator
+from src.data_handling.data_generator import DataGenerator
+from src.helpers import PyTestAllEqual, xr_all_equal
-import pytest
+import logging
+import mock
import os
+import pytest
+import shutil
+import typing
import numpy as np
+import xarray as xr
+
+
+@pytest.fixture
+def orig_generator(data_path):
+ return DataGenerator(data_path, 'AIRBASE', ['DEBW107', 'DEBW013'],
+ ['o3', 'temp'], 'datetime', 'variables', 'o3', start=2010, end=2014,
+ statistics_per_var={"o3": "dma8eu", "temp": "maximum"})
-class TestBootstraps:
+@pytest.fixture
+def data_path():
+ path = os.path.join(os.path.dirname(__file__), "data")
+ if not os.path.exists(path):
+ os.makedirs(path)
+ return path
+
+
+class TestBootStrapGenerator:
@pytest.fixture
- def path(self):
- path = os.path.join(os.path.dirname(__file__), "data")
- if not os.path.exists(path):
- os.makedirs(path)
- return path
+ def hist(self, orig_generator):
+ return orig_generator.get_data_generator(0).get_transposed_history()
@pytest.fixture
- def boot_no_init(self, path):
- obj = object.__new__(BootStraps)
- super(BootStraps, obj).__init__()
- obj.number_bootstraps = 50
- obj.bootstrap_path = path
- return obj
-
- def test_valid_bootstrap_file(self, path, boot_no_init):
- station = "TESTSTATION"
- variables = "var1_var2_var3"
- window = 5
- # empty case
- assert len(os.listdir(path)) == 0
- assert boot_no_init.valid_bootstrap_file(station, variables, window) == (False, 50)
- # different cases, where files with bigger range are existing
- os.mknod(os.path.join(path, f"{station}_{variables}_hist5_nboots50_shuffled.dat"))
- assert boot_no_init.valid_bootstrap_file(station, variables, window) == (True, None)
- os.mknod(os.path.join(path, f"{station}_{variables}_hist5_nboots100_shuffled.dat"))
- assert boot_no_init.valid_bootstrap_file(station, variables, window) == (True, None)
- os.mknod(os.path.join(path, f"{station}_{variables}_hist10_nboots50_shuffled.dat"))
- os.mknod(os.path.join(path, f"{station}1_{variables}_hist10_nboots50_shuffled.dat"))
- assert boot_no_init.valid_bootstrap_file(station, variables, window) == (True, None)
- # need to reload data and therefore remove not fitting files for this station
- assert boot_no_init.valid_bootstrap_file(station, variables, 20) == (False, 100)
- assert len(os.listdir(path)) == 1
+ def boot_gen(self, hist):
+ return BootStrapGenerator(20, hist, hist.expand_dims({"boots": [0]}) + 1, ["o3", "temp"], "o3")
+
+ def test_init(self, boot_gen, hist):
+ assert boot_gen.number_of_boots == 20
+ assert boot_gen.variables == ["o3", "temp"]
+ assert xr.testing.assert_equal(boot_gen.history_orig, hist) is None
+ assert xr.testing.assert_equal(boot_gen.history, hist.sel(variables=["temp"])) is None
+ assert xr.testing.assert_allclose(boot_gen.shuffled - 1, hist.sel(variables="o3").expand_dims({"boots": [0]})) is None
+
+ def test_len(self, boot_gen):
+ assert len(boot_gen) == 20
+
+ def test_get_shuffled(self, boot_gen, hist):
+ shuffled = boot_gen._BootStrapGenerator__get_shuffled(0)
+ expected = hist.sel(variables=["o3"]).transpose("datetime", "window", "Stations", "variables") + 1
+ assert xr.testing.assert_equal(shuffled, expected) is None
+
+ def test_getitem(self, boot_gen, hist):
+ first_element = boot_gen[0]
+ assert xr.testing.assert_equal(first_element.sel(variables="temp"), hist.sel(variables="temp")) is None
+ assert xr.testing.assert_allclose(first_element.sel(variables="o3"), hist.sel(variables="o3") + 1) is None
+
+ def test_next(self, boot_gen, hist):
+ iter_obj = iter(boot_gen)
+ first_element = next(iter_obj)
+ assert xr.testing.assert_equal(first_element.sel(variables="temp"), hist.sel(variables="temp")) is None
+ assert xr.testing.assert_allclose(first_element.sel(variables="o3"), hist.sel(variables="o3") + 1) is None
+ with pytest.raises(KeyError):
+ next(iter_obj)
+
+
+class TestCreateShuffledData:
+
+ @pytest.fixture
+ def shuffled_data(self, orig_generator, data_path):
+ return CreateShuffledData(orig_generator, 20, data_path)
+
+ @pytest.fixture
+ @mock.patch("src.data_handling.bootstraps.CreateShuffledData.create_shuffled_data", return_value=None)
+ def shuffled_data_no_creation(self, mock_create_shuffle_data, orig_generator, data_path):
+ return CreateShuffledData(orig_generator, 20, data_path)
+
+ @pytest.fixture
+ def shuffled_data_clean(self, shuffled_data_no_creation):
+ shutil.rmtree(shuffled_data_no_creation.bootstrap_path)
+ os.makedirs(shuffled_data_no_creation.bootstrap_path)
+ assert os.listdir(shuffled_data_no_creation.bootstrap_path) == [] # just to check for a clean working directory
+ return shuffled_data_no_creation
+
+ def test_init(self, shuffled_data_no_creation, data_path):
+ assert isinstance(shuffled_data_no_creation.data, DataGenerator)
+ assert shuffled_data_no_creation.number_of_bootstraps == 20
+ assert shuffled_data_no_creation.bootstrap_path == data_path
+
+ def test_create_shuffled_data_create_new(self, shuffled_data_clean, data_path, caplog):
+ caplog.set_level(logging.INFO)
+ shuffled_data_clean.data.data_path_tmp = data_path
+ assert shuffled_data_clean.create_shuffled_data() is None
+ assert caplog.record_tuples[0] == ('root', logging.INFO, "create / check shuffled bootstrap data")
+ assert caplog.record_tuples[1] == ('root', logging.INFO, "create bootstap data for DEBW107")
+ assert caplog.record_tuples[5] == ('root', logging.INFO, "create bootstap data for DEBW013")
+ assert "DEBW107_o3_temp_hist7_nboots20_shuffled.nc" in os.listdir(data_path)
+ assert "DEBW013_o3_temp_hist7_nboots20_shuffled.nc" in os.listdir(data_path)
+
+ def test_create_shuffled_data_some_valid(self, shuffled_data_clean, data_path, caplog):
+ shuffled_data_clean.data.data_path_tmp = data_path
+ shuffled_data_clean.create_shuffled_data()
+ caplog.records.clear()
+ caplog.set_level(logging.INFO)
+ os.rename(os.path.join(data_path, "DEBW013_o3_temp_hist7_nboots20_shuffled.nc"),
+ os.path.join(data_path, "DEBW013_o3_temp_hist5_nboots30_shuffled.nc"))
+ shuffled_data_clean.create_shuffled_data()
+ assert caplog.record_tuples[0] == ('root', logging.INFO, "create / check shuffled bootstrap data")
+ assert caplog.record_tuples[1] == ('root', logging.INFO, "create bootstap data for DEBW013")
+ assert "DEBW107_o3_temp_hist7_nboots20_shuffled.nc" in os.listdir(data_path)
+ assert "DEBW013_o3_temp_hist7_nboots30_shuffled.nc" in os.listdir(data_path)
+ assert "DEBW013_o3_temp_hist5_nboots30_shuffled.nc" not in os.listdir(data_path)
+
+ def test_set_file_path(self, shuffled_data_no_creation):
+ res = shuffled_data_no_creation._set_file_path("DEBWtest", "o3_temp_wind", 10, 5)
+ assert "DEBWtest_o3_temp_wind_hist10_nboots5_shuffled.nc" in res
+ assert shuffled_data_no_creation.bootstrap_path in res
+
+ def test_valid_bootstrap_file_blank(self, shuffled_data_clean):
+ assert shuffled_data_clean.valid_bootstrap_file("DEBWtest", "o3_temp", 10) == (False, 20)
+
+ def test_valid_bootstrap_file_already_satisfied(self, shuffled_data_clean, data_path):
+ station, variables, window = "DEBWtest2", "o3_temp", 5
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist5_nboots50_shuffled.dat"))
+ assert shuffled_data_clean.valid_bootstrap_file(station, variables, window) == (True, None)
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist5_nboots100_shuffled.dat"))
+ assert shuffled_data_clean.valid_bootstrap_file(station, variables, window) == (True, None)
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist10_nboots50_shuffled.dat"))
+ os.mknod(os.path.join(data_path, f"{station}1_{variables}_hist10_nboots50_shuffled.dat"))
+ assert shuffled_data_clean.valid_bootstrap_file(station, variables, window) == (True, None)
+
+ def test_valid_bootstrap_file_reload_data_window(self, shuffled_data_clean, data_path):
+ station, variables, window = "DEBWtest2", "o3_temp", 20
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist5_nboots50_shuffled.dat"))
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist5_nboots100_shuffled.dat"))
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist10_nboots50_shuffled.dat"))
+ os.mknod(os.path.join(data_path, f"{station}1_{variables}_hist10_nboots50_shuffled.dat")) # <- DEBWtest21
+ # need to reload data and therefore remove not fitting history size in all files for this station
+ assert shuffled_data_clean.valid_bootstrap_file(station, variables, window) == (False, 100)
+ assert len(os.listdir(data_path)) == 1 # keep only data from other station DEBWtest21
+
+ def test_valid_bootstrap_file_reload_data_boots(self, shuffled_data_clean, data_path):
+ station, variables, window = "DEBWtest2", "o3_temp", 5
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist5_nboots50_shuffled.dat"))
+ os.mknod(os.path.join(data_path, f"{station}1_{variables}_hist10_nboots50_shuffled.dat")) # <- DEBWtest21
# reload because expanded boot number
- os.mknod(os.path.join(path, f"{station}_{variables}_hist5_nboots50_shuffled.dat"))
- boot_no_init.number_bootstraps = 60
- assert boot_no_init.valid_bootstrap_file(station, variables, window) == (False, 60)
- assert len(os.listdir(path)) == 1
+ shuffled_data_clean.number_of_bootstraps = 60
+ assert shuffled_data_clean.valid_bootstrap_file(station, variables, window) == (False, 60)
+ assert len(os.listdir(data_path)) == 1
+
+ def test_valid_bootstrap_file_reload_data_use_max_file_boot(self, shuffled_data_clean, data_path):
+ station, variables, window = "DEBWtest2", "o3_temp", 20
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist5_nboots50_shuffled.dat"))
+ os.mknod(os.path.join(data_path, f"{station}_{variables}_hist5_nboots60_shuffled.dat"))
+ os.mknod(os.path.join(data_path, f"{station}1_{variables}_hist10_nboots50_shuffled.dat")) # <- DEBWtest21
# reload because of expanded window size, but use maximum boot number from file names
- os.mknod(os.path.join(path, f"{station}_{variables}_hist5_nboots60_shuffled.dat"))
- boot_no_init.number_bootstraps = 50
- assert boot_no_init.valid_bootstrap_file(station, variables, 20) == (False, 60)
-
- def test_shuffle_single_variale(self, boot_no_init):
- data = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3]])
- res = boot_no_init.shuffle_single_variable(data, chunks=(2, 3)).compute()
- assert res.shape == data.shape
- assert res.max() == data.max()
- assert res.min() == data.min()
+ assert shuffled_data_clean.valid_bootstrap_file(station, variables, window) == (False, 60)
+
+ def test_shuffle(self, shuffled_data_no_creation):
+ dummy = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2, 3]])
+ res = shuffled_data_no_creation.shuffle(dummy, chunks=(2, 3)).compute()
+ assert res.shape == dummy.shape
+ assert dummy.max() >= res.max()
+ assert dummy.min() <= res.min()
assert set(np.unique(res)).issubset({1, 2, 3})
- def test_create_shuffled_data(self):
- pass
\ No newline at end of file
+
+class TestBootStraps:
+
+ @pytest.fixture
+ def bootstrap(self, orig_generator, data_path):
+ return BootStraps(orig_generator, data_path, 20)
+
+ @pytest.fixture
+ @mock.patch("src.data_handling.bootstraps.CreateShuffledData", return_value=None)
+ def bootstrap_no_shuffling(self, mock_create_shuffle_data, orig_generator, data_path):
+ shutil.rmtree(data_path)
+ return BootStraps(orig_generator, data_path, 20)
+
+ def test_init_no_shuffling(self, bootstrap_no_shuffling, data_path):
+ assert isinstance(bootstrap_no_shuffling, BootStraps)
+ assert bootstrap_no_shuffling.number_of_bootstraps == 20
+ assert bootstrap_no_shuffling.bootstrap_path == data_path
+
+ def test_init_with_shuffling(self, orig_generator, data_path, caplog):
+ caplog.set_level(logging.INFO)
+ BootStraps(orig_generator, data_path, 20)
+ assert caplog.record_tuples[0] == ('root', logging.INFO, "create / check shuffled bootstrap data")
+
+ def test_stations(self, bootstrap_no_shuffling, orig_generator):
+ assert bootstrap_no_shuffling.stations == orig_generator.stations
+
+ def test_variables(self, bootstrap_no_shuffling, orig_generator):
+ assert bootstrap_no_shuffling.variables == orig_generator.variables
+
+ def test_window_history_size(self, bootstrap_no_shuffling, orig_generator):
+ assert bootstrap_no_shuffling.window_history_size == orig_generator.window_history_size
+
+ def test_get_generator(self, bootstrap, orig_generator):
+ station = bootstrap.stations[0]
+ var = bootstrap.variables[0]
+ var_others = bootstrap.variables[1:]
+ gen = bootstrap.get_generator(station, var)
+ assert isinstance(gen, BootStrapGenerator)
+ assert gen.number_of_boots == bootstrap.number_of_bootstraps
+ assert gen.variables == bootstrap.variables
+ expected = orig_generator.get_data_generator(station).get_transposed_history()
+ assert xr.testing.assert_equal(gen.history_orig, expected) is None
+ assert xr.testing.assert_equal(gen.history, expected.sel(variables=var_others)) is None
+ assert gen.shuffled.variables == "o3"
+
+ @mock.patch("src.data_handling.data_generator.DataGenerator._load_pickle_data", side_effect=FileNotFoundError)
+ def test_get_generator_different_generator(self, mock_load_pickle, data_path, orig_generator):
+ BootStraps(orig_generator, data_path, 20) # to create
+ orig_generator.window_history_size = 4
+ bootstrap = BootStraps(orig_generator, data_path, 20)
+ station = bootstrap.stations[0]
+ var = bootstrap.variables[0]
+ var_others = bootstrap.variables[1:]
+ gen = bootstrap.get_generator(station, var)
+ expected = orig_generator.get_data_generator(station, load_local_tmp_storage=False).get_transposed_history()
+ assert xr.testing.assert_equal(gen.history_orig, expected) is None
+ assert xr.testing.assert_equal(gen.history, expected.sel(variables=var_others)) is None
+ assert gen.shuffled.variables == "o3"
+ assert gen.shuffled.shape[:-1] == expected.shape[:-1]
+ assert gen.shuffled.shape[-1] == 20
+
+ def test_get_labels(self, bootstrap, orig_generator):
+ station = bootstrap.stations[0]
+ labels = bootstrap.get_labels(station)
+ labels_orig = orig_generator.get_data_generator(station).get_transposed_label()
+ assert labels.shape == (labels_orig.shape[0] * bootstrap.number_of_bootstraps, *labels_orig.shape[1:])
+ assert np.testing.assert_array_equal(labels[:labels_orig.shape[0], :], labels_orig.values) is None
+
+ def test_get_orig_prediction(self, bootstrap, data_path, orig_generator):
+ station = bootstrap.stations[0]
+ labels = orig_generator.get_data_generator(station).get_transposed_label()
+ predictions = labels.expand_dims({"type": ["CNN"]}, -1)
+ file_name = "test_prediction.nc"
+ predictions.to_netcdf(os.path.join(data_path, file_name))
+ res = bootstrap.get_orig_prediction(data_path, file_name)
+ assert (*res.shape, 1) == (predictions.shape[0] * bootstrap.number_of_bootstraps, *predictions.shape[1:])
+ assert np.testing.assert_array_equal(res[:predictions.shape[0], :], predictions.squeeze().values) is None
+
+ def test_load_shuffled_data(self, bootstrap, orig_generator):
+ station = bootstrap.stations[0]
+ hist = orig_generator.get_data_generator(station).get_transposed_history()
+ shuffled_data = bootstrap._load_shuffled_data(station, ["o3", "temp"])
+ assert isinstance(shuffled_data, xr.DataArray)
+ assert hist.shape[0] >= shuffled_data.shape[0] # longer window length lead to shorter datetime axis in shuffled
+ assert hist.shape[1] <= shuffled_data.shape[1] # longer window length in shuffled
+ assert hist.shape[2] == shuffled_data.shape[2]
+ assert hist.shape[3] <= shuffled_data.shape[3] # potentially more variables in shuffled
+ assert bootstrap.number_of_bootstraps == shuffled_data.shape[4]
+ assert shuffled_data.mean().compute()
+ assert np.testing.assert_almost_equal(shuffled_data.mean().compute(), hist.mean(), decimal=1) is None
+ assert shuffled_data.max() <= hist.max()
+ assert shuffled_data.min() >= hist.min()
+
+ def test_get_shuffled_data_file(self, bootstrap):
+ file_name = bootstrap._get_shuffled_data_file("DEBW107", ["o3"])
+ assert file_name == os.path.join(bootstrap.bootstrap_path, "DEBW107_o3_temp_hist7_nboots20_shuffled.nc")
+
+ def test_get_shuffled_data_file_not_found(self, bootstrap_no_shuffling, data_path):
+ bootstrap_no_shuffling.number_of_boots = 100
+ os.makedirs(data_path)
+ with pytest.raises(FileNotFoundError) as e:
+ bootstrap_no_shuffling._get_shuffled_data_file("DEBW107", ["o3"])
+ assert "Could not find a file to match pattern" in e.value.args[0]
+
+ def test_create_file_regex(self, bootstrap_no_shuffling):
+ regex = bootstrap_no_shuffling._create_file_regex("DEBW108", ["o3", "temp", "h2o"])
+ assert regex.match("DEBW108_h2o_hum_latent_o3_temp_h20_hist10_nboots10_shuffled.nc")
+ regex.match("DEBW108_h2o_hum_latent_o3_temp_hist10_shuffled.nc") is None
+
+ def test_filter_files(self, bootstrap_no_shuffling):
+ regex = bootstrap_no_shuffling._create_file_regex("DEBW108", ["o3", "temp", "h2o"])
+ test_list = ["DEBW108_o3_test23_test_shuffled.nc",
+ "DEBW107_o3_test23_test_shuffled.nc",
+ "DEBW108_o3_test23_test.nc",
+ "DEBW108_h2o_o3_temp_test_shuffled.nc",
+ "DEBW108_h2o_hum_latent_o3_temp_u_v_test23_test_shuffled.nc",
+ "DEBW108_o3_temp_hist9_nboots20_shuffled.nc",
+ "DEBW108_h2o_o3_temp_hist9_nboots20_shuffled.nc"]
+ f = bootstrap_no_shuffling._filter_files
+ assert f(regex, test_list, 10, 10) is None
+ assert f(regex, test_list, 9, 10) == "DEBW108_h2o_o3_temp_hist9_nboots20_shuffled.nc"
+ assert f(regex, test_list, 9, 20) == "DEBW108_h2o_o3_temp_hist9_nboots20_shuffled.nc"
+
diff --git a/test/test_data_handling/test_data_distributor.py b/test/test_data_handling/test_data_distributor.py
index a26e76a0e7f3ef0f5cdbedc07d73a690116966c9..15344fd808a4aa9ee5774ad8ba647bf5ce06d015 100644
--- a/test/test_data_handling/test_data_distributor.py
+++ b/test/test_data_handling/test_data_distributor.py
@@ -46,7 +46,7 @@ class TestDistributor:
distributor.model = 1
def test_get_number_of_mini_batches(self, distributor):
- values = np.zeros((2, 2311, 19))
+ values = np.zeros((2311, 19))
assert distributor._get_number_of_mini_batches(values) == math.ceil(2311 / distributor.batch_size)
def test_distribute_on_batches_single_loop(self, generator_two_stations, model):
@@ -98,3 +98,21 @@ class TestDistributor:
assert np.testing.assert_equal(x, x_perm) is None
assert np.testing.assert_equal(y, y_perm) is None
+ def test_distribute_on_batches_upsampling_no_extremes_given(self, generator, model):
+ d = Distributor(generator, model, upsampling=True)
+ gen_len = d.generator.get_data_generator(0, load_local_tmp_storage=False).get_transposed_label().shape[0]
+ num_mini_batches = math.ceil(gen_len / d.batch_size)
+ i = 0
+ for i, e in enumerate(d.distribute_on_batches(fit_call=False)):
+ assert e[0].shape[0] <= d.batch_size
+ assert i + 1 == num_mini_batches
+
+ def test_distribute_on_batches_upsampling(self, generator, model):
+ generator.extreme_values = [1]
+ d = Distributor(generator, model, upsampling=True)
+ gen_len = d.generator.get_data_generator(0, load_local_tmp_storage=False).get_transposed_label().shape[0]
+ extr_len = d.generator.get_data_generator(0, load_local_tmp_storage=False).get_extremes_label().shape[0]
+ i = 0
+ for i, e in enumerate(d.distribute_on_batches(fit_call=False)):
+ assert e[0].shape[0] <= d.batch_size
+ assert i + 1 == math.ceil((gen_len + extr_len) / d.batch_size)
diff --git a/test/test_data_handling/test_data_generator.py b/test/test_data_handling/test_data_generator.py
index 9bf11154609afa9ada2b488455f7a341a41d21ae..939f93cc9ee01c76a282e755aca14b39c6fc4ac9 100644
--- a/test/test_data_handling/test_data_generator.py
+++ b/test/test_data_handling/test_data_generator.py
@@ -238,6 +238,20 @@ class TestDataGenerator:
assert data._transform_method == "standardise"
assert data.mean is not None
+ def test_get_data_generator_extremes(self, gen_with_transformation):
+ gen = gen_with_transformation
+ gen.kwargs = {"statistics_per_var": {'o3': 'dma8eu', 'temp': 'maximum'}}
+ gen.extreme_values = [1.]
+ data = gen.get_data_generator("DEBW107", load_local_tmp_storage=False, save_local_tmp_storage=False)
+ assert data.extremes_label is not None
+ assert data.extremes_history is not None
+ assert data.extremes_label.shape[:2] == data.label.shape[:2]
+ assert data.extremes_label.shape[2] <= data.label.shape[2]
+ len_both_tails = data.extremes_label.shape[2]
+ gen.kwargs["extremes_on_right_tail_only"] = True
+ data = gen.get_data_generator("DEBW107", load_local_tmp_storage=False, save_local_tmp_storage=False)
+ assert data.extremes_label.shape[2] <= len_both_tails
+
def test_save_pickle_data(self, gen):
file = os.path.join(gen.data_path_tmp, f"DEBW107_{'_'.join(sorted(gen.variables))}_2010_2014_.pickle")
if os.path.exists(file):
diff --git a/test/test_data_handling/test_data_preparation.py b/test/test_data_handling/test_data_preparation.py
index 91719f3dd16326ee6281c4db8ef3aa87e238d70f..747b3734f565d3206696998de10f5986b7c94bf0 100644
--- a/test/test_data_handling/test_data_preparation.py
+++ b/test/test_data_handling/test_data_preparation.py
@@ -1,6 +1,6 @@
import datetime as dt
import os
-from operator import itemgetter
+from operator import itemgetter, lt, gt
import logging
import numpy as np
@@ -287,6 +287,14 @@ class TestDataPrep:
assert remaining_len == data.label.datetime.shape
assert remaining_len == data.observation.datetime.shape
+ def test_remove_nan_too_short(self, data):
+ data.kwargs["min_length"] = 4000 # actual length of series is 3940
+ data.make_history_window('variables', -12, 'datetime')
+ data.make_labels('variables', 'o3', 'datetime', 3)
+ data.make_observation('variables', 'o3', 'datetime')
+ data.remove_nan('datetime')
+ assert not any([data.history, data.label, data.observation])
+
def test_create_index_array(self, data):
index_array = data.create_index_array('window', range(1, 4))
assert np.testing.assert_array_equal(index_array.data, [1, 2, 3]) is None
@@ -395,3 +403,84 @@ class TestDataPrep:
data.make_labels("variables", "o3", "datetime", 2)
transposed = data.get_transposed_label()
assert transposed.coords.dims == ("datetime", "window")
+
+ def test_multiply_extremes(self, data):
+ data.transform("datetime")
+ data.make_history_window("variables", 3, "datetime")
+ data.make_labels("variables", "o3", "datetime", 2)
+ orig = data.label
+ data.multiply_extremes(1)
+ upsampled = data.extremes_label
+ assert (upsampled > 1).sum() == (orig > 1).sum()
+ assert (upsampled < -1).sum() == (orig < -1).sum()
+
+ def test_multiply_extremes_from_list(self, data):
+ data.transform("datetime")
+ data.make_history_window("variables", 3, "datetime")
+ data.make_labels("variables", "o3", "datetime", 2)
+ orig = data.label
+ data.multiply_extremes([1, 1.5, 2, 3])
+ upsampled = data.extremes_label
+ def f(d, op, n):
+ return op(d, n).any(dim="window").sum()
+ assert f(upsampled, gt, 1) == sum([f(orig, gt, 1), f(orig, gt, 1.5), f(orig, gt, 2) * 2, f(orig, gt, 3) * 4])
+ assert f(upsampled, lt, -1) == sum([f(orig, lt, -1), f(orig, lt, -1.5), f(orig, lt, -2) * 2, f(orig, lt, -3) * 4])
+
+ def test_multiply_extremes_wrong_extremes(self, data):
+ data.transform("datetime")
+ data.make_history_window("variables", 3, "datetime")
+ data.make_labels("variables", "o3", "datetime", 2)
+ with pytest.raises(TypeError) as e:
+ data.multiply_extremes([1, "1.5", 2])
+ assert "Elements of list extreme_values have to be (<class 'float'>, <class 'int'>), but at least element 1.5" \
+ " is type <class 'str'>" in e.value.args[0]
+
+ def test_multiply_extremes_right_tail(self, data):
+ data.transform("datetime")
+ data.make_history_window("variables", 3, "datetime")
+ data.make_labels("variables", "o3", "datetime", 2)
+ orig = data.label
+ data.multiply_extremes([1, 2], extremes_on_right_tail_only=True)
+ upsampled = data.extremes_label
+ def f(d, op, n):
+ return op(d, n).any(dim="window").sum()
+ assert f(upsampled, gt, 1) == sum([f(orig, gt, 1), f(orig, gt, 2)])
+ assert upsampled.shape[2] == sum([f(orig, gt, 1), f(orig, gt, 2)])
+ assert f(upsampled, lt, -1) == 0
+
+ def test_multiply_extremes_none_label(self, data):
+ data.transform("datetime")
+ data.make_history_window("variables", 3, "datetime")
+ data.label = None
+ assert data.multiply_extremes([1], extremes_on_right_tail_only=False) is None
+
+ def test_multiply_extremes_none_history(self,data ):
+ data.transform("datetime")
+ data.history = None
+ data.make_labels("variables", "o3", "datetime", 2)
+ assert data.multiply_extremes([1], extremes_on_right_tail_only=False) is None
+
+ def test_multiply_extremes_none_label_history(self,data ):
+ data.history = None
+ data.label = None
+ assert data.multiply_extremes([1], extremes_on_right_tail_only=False) is None
+
+ def test_get_extremes_history(self, data):
+ data.transform("datetime")
+ data.make_history_window("variables", 3, "datetime")
+ data.make_labels("variables", "o3", "datetime", 2)
+ data.make_observation("variables", "o3", "datetime")
+ data.remove_nan("datetime")
+ data.multiply_extremes([1, 2], extremes_on_right_tail_only=True)
+ assert (data.get_extremes_history() ==
+ data.extremes_history.transpose("datetime", "window", "Stations", "variables")).all()
+
+ def test_get_extremes_label(self, data):
+ data.transform("datetime")
+ data.make_history_window("variables", 3, "datetime")
+ data.make_labels("variables", "o3", "datetime", 2)
+ data.make_observation("variables", "o3", "datetime")
+ data.remove_nan("datetime")
+ data.multiply_extremes([1, 2], extremes_on_right_tail_only=True)
+ assert (data.get_extremes_label() ==
+ data.extremes_label.squeeze("Stations").transpose("datetime", "window")).all()
diff --git a/test/test_datastore.py b/test/test_datastore.py
index 9fcb319f51954b365c59274a4a9744f093e155f1..5b6cd17a00271a17b8fe5c30ca26665b42e56141 100644
--- a/test/test_datastore.py
+++ b/test/test_datastore.py
@@ -4,7 +4,7 @@ __date__ = '2019-11-22'
import pytest
-from src.datastore import AbstractDataStore, DataStoreByVariable, DataStoreByScope
+from src.datastore import AbstractDataStore, DataStoreByVariable, DataStoreByScope, CorrectScope
from src.datastore import NameNotFoundInDataStore, NameNotFoundInScope, EmptyScope
@@ -68,7 +68,7 @@ class TestDataStoreByVariable:
ds.set("number", 3, "general")
assert ds.get_default("number", "general", 45) == 3
assert ds.get_default("number", "general.sub", 45) == 3
- assert ds.get_default("number", "other", 45) == 45
+ assert ds.get_default("other", 45) == 45
def test_search(self, ds):
ds.set("number", 22, "general")
@@ -161,6 +161,19 @@ class TestDataStoreByVariable:
assert ds.get("tester1", "general.sub") == 111
assert ds.get("tester3", "general.sub") == 21
+ def test_no_scope_given(self, ds):
+ ds.set("tester", 34)
+ assert ds._store["tester"]["general"] == 34
+ assert ds.get("tester") == 34
+ assert ds.get("tester", "sub") == 34
+ ds.set("tester", 99, "sub")
+ assert ds.list_all_scopes() == ["general", "general.sub"]
+ assert ds.get_default("test2", 4) == 4
+ assert ds.get_default("tester", "sub", 4) == 99
+ ds.set("test2", 4)
+ assert sorted(ds.search_scope(current_scope_only=False)) == sorted(["tester", "test2"])
+ assert ds.search_scope("sub", current_scope_only=True) == ["tester"]
+
class TestDataStoreByScope:
@@ -206,7 +219,7 @@ class TestDataStoreByScope:
ds.set("number", 3, "general")
assert ds.get_default("number", "general", 45) == 3
assert ds.get_default("number", "general.sub", 45) == 3
- assert ds.get_default("number", "other", 45) == 45
+ assert ds.get_default("other", "other", 45) == 45
def test_search(self, ds):
ds.set("number", 22, "general")
@@ -297,4 +310,31 @@ class TestDataStoreByScope:
assert ds.get("tester3", "general") == 21
ds.set_args_from_dict({"tester1": 111}, "general.sub")
assert ds.get("tester1", "general.sub") == 111
- assert ds.get("tester3", "general.sub") == 21
\ No newline at end of file
+ assert ds.get("tester3", "general.sub") == 21
+
+ def test_no_scope_given(self, ds):
+ ds.set("tester", 34)
+ assert ds._store["general"]["tester"] == 34
+ assert ds.get("tester") == 34
+ assert ds.get("tester", "sub") == 34
+ ds.set("tester", 99, "sub")
+ assert ds.list_all_scopes() == ["general", "general.sub"]
+ assert ds.get_default("test2", 4) == 4
+ assert ds.get_default("tester", "sub", 4) == 99
+ ds.set("test2", 4)
+ assert sorted(ds.search_scope(current_scope_only=False)) == sorted(["tester", "test2"])
+ assert ds.search_scope("sub", current_scope_only=True) == ["tester"]
+
+
+class TestCorrectScope:
+
+ @staticmethod
+ @CorrectScope
+ def function1(a, scope, b=44):
+ return a, scope, b
+
+ def test_init(self):
+ assert self.function1(22, "general") == (22, "general", 44)
+ assert self.function1(21) == (21, "general", 44)
+ assert self.function1(55, "sub", 34) == (55, "general.sub", 34)
+ assert self.function1("string", b=99, scope="tester") == ("string", "general.tester", 99)
diff --git a/test/test_helpers.py b/test/test_helpers.py
index 07ec244e078f977dca761274260275aab355c183..9c71a53389344083e4e18a83a6aab5838ad678ca 100644
--- a/test/test_helpers.py
+++ b/test/test_helpers.py
@@ -7,6 +7,8 @@ import mock
import numpy as np
import pytest
+import re
+
from src.helpers import *
@@ -128,32 +130,63 @@ class TestTimeTracking:
class TestPrepareHost:
- @mock.patch("socket.gethostname", side_effect=["linux-aa9b", "ZAM144", "zam347", "jrtest", "jwtest"])
+ @mock.patch("socket.gethostname", side_effect=["linux-aa9b", "ZAM144", "zam347", "jrtest", "jwtest",
+ "runner-6HmDp9Qd-project-2411-concurrent-01"])
@mock.patch("os.getlogin", return_value="testUser")
@mock.patch("os.path.exists", return_value=True)
def test_prepare_host(self, mock_host, mock_user, mock_path):
- path = prepare_host()
- assert path == "/home/testUser/machinelearningtools/data/toar_daily/"
- path = prepare_host()
- assert path == "/home/testUser/Data/toar_daily/"
- path = prepare_host()
- assert path == "/home/testUser/Data/toar_daily/"
- path = prepare_host()
- assert path == "/p/project/cjjsc42/testUser/DATA/toar_daily/"
- path = prepare_host()
- assert path == "/p/home/jusers/testUser/juwels/intelliaq/DATA/toar_daily/"
+ assert prepare_host() == "/home/testUser/machinelearningtools/data/toar_daily/"
+ assert prepare_host() == "/home/testUser/Data/toar_daily/"
+ assert prepare_host() == "/home/testUser/Data/toar_daily/"
+ assert prepare_host() == "/p/project/cjjsc42/testUser/DATA/toar_daily/"
+ assert prepare_host() == "/p/home/jusers/testUser/juwels/intelliaq/DATA/toar_daily/"
+ assert prepare_host() == '/home/testUser/machinelearningtools/data/toar_daily/'
@mock.patch("socket.gethostname", return_value="NotExistingHostName")
@mock.patch("os.getlogin", return_value="zombie21")
- def test_error_handling(self, mock_user, mock_host):
+ def test_error_handling_unknown_host(self, mock_user, mock_host):
with pytest.raises(OSError) as e:
prepare_host()
assert "unknown host 'NotExistingHostName'" in e.value.args[0]
- if "runner-6HmDp9Qd-project-2411-concurrent" not in platform.node():
- mock_host.return_value = "linux-aa9b"
- with pytest.raises(NotADirectoryError) as e:
- prepare_host()
- assert "does not exist for host 'linux-aa9b'" in e.value.args[0]
+
+ @mock.patch("os.getlogin", return_value="zombie21")
+ @mock.patch("src.helpers.check_path_and_create", side_effect=PermissionError)
+ def test_error_handling(self, mock_cpath, mock_user):
+ # if "runner-6HmDp9Qd-project-2411-concurrent" not in platform.node():
+ # mock_host.return_value = "linux-aa9b"
+ with pytest.raises(NotADirectoryError) as e:
+ prepare_host()
+ assert PyTestRegex(r"path '.*' does not exist for host '.*'\.") == e.value.args[0]
+ with pytest.raises(NotADirectoryError) as e:
+ prepare_host(False)
+ # assert "does not exist for host 'linux-aa9b'" in e.value.args[0]
+ assert PyTestRegex(r"path '.*' does not exist for host '.*'\.") == e.value.args[0]
+
+ @mock.patch("socket.gethostname", side_effect=["linux-aa9b", "ZAM144", "zam347", "jrtest", "jwtest",
+ "runner-6HmDp9Qd-project-2411-concurrent-01"])
+ @mock.patch("os.getlogin", side_effect=OSError)
+ @mock.patch("os.path.exists", return_value=True)
+ def test_os_error(self, mock_path, mock_user, mock_host):
+ path = prepare_host()
+ assert path == "/home/default/machinelearningtools/data/toar_daily/"
+ path = prepare_host()
+ assert path == "/home/default/Data/toar_daily/"
+ path = prepare_host()
+ assert path == "/home/default/Data/toar_daily/"
+ path = prepare_host()
+ assert path == "/p/project/cjjsc42/default/DATA/toar_daily/"
+ path = prepare_host()
+ assert path == "/p/home/jusers/default/juwels/intelliaq/DATA/toar_daily/"
+ path = prepare_host()
+ assert path == '/home/default/machinelearningtools/data/toar_daily/'
+
+ @mock.patch("socket.gethostname", side_effect=["linux-aa9b"])
+ @mock.patch("os.getlogin", return_value="testUser")
+ @mock.patch("os.path.exists", return_value=False)
+ @mock.patch("os.makedirs", side_effect=None)
+ def test_os_path_exists(self, mock_host, mock_user, mock_path, mock_check):
+ path = prepare_host()
+ assert path == "/home/testUser/machinelearningtools/data/toar_daily/"
class TestSetExperimentName:
@@ -170,6 +203,23 @@ class TestSetExperimentName:
exp_name, _ = set_experiment_name(experiment_date="2019-11-14")
assert exp_name == "2019-11-14_network"
+ def test_set_expperiment_hourly(self):
+ exp_name, exp_path = set_experiment_name(sampling="hourly")
+ assert exp_name == "TestExperiment_hourly"
+ assert exp_path == os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "TestExperiment_hourly"))
+
+
+class TestSetBootstrapPath:
+
+ def test_bootstrap_path_is_none(self):
+ bootstrap_path = set_bootstrap_path(None, 'TestDataPath/', 'daily')
+ assert bootstrap_path == 'TestDataPath/../bootstrap_daily'
+
+ @mock.patch("os.makedirs", side_effect=None)
+ def test_bootstap_path_is_given(self, mock_makedir):
+ bootstrap_path = set_bootstrap_path('Test/path/to/boots', None, None)
+ assert bootstrap_path == 'Test/path/to/boots'
+
class TestPytestRegex:
@@ -221,3 +271,123 @@ class TestFloatRound:
assert float_round(-34.9221, 0, math.floor) == -35.
assert float_round(-34.9221, 2) == -34.92
assert float_round(-34.9221, 0) == -34.
+
+
+class TestDictPop:
+
+ @pytest.fixture
+ def custom_dict(self):
+ return {'a': 1, 'b': 2, 2: 'ab'}
+
+ def test_dict_pop_single(self, custom_dict):
+ # one out as list
+ d_pop = dict_pop(custom_dict, [4])
+ assert d_pop == custom_dict
+ # one out as str
+ d_pop = dict_pop(custom_dict, '4')
+ assert d_pop == custom_dict
+ # one in as str
+ d_pop = dict_pop(custom_dict, 'b')
+ assert d_pop == {'a': 1, 2: 'ab'}
+ # one in as list
+ d_pop = dict_pop(custom_dict, ['b'])
+ assert d_pop == {'a': 1, 2: 'ab'}
+
+ def test_dict_pop_multiple(self, custom_dict):
+ # all out (list)
+ d_pop = dict_pop(custom_dict, [4, 'mykey'])
+ assert d_pop == custom_dict
+ # all in (list)
+ d_pop = dict_pop(custom_dict, ['a', 2])
+ assert d_pop == {'b': 2}
+ # one in one out (list)
+ d_pop = dict_pop(custom_dict, [2, '10'])
+ assert d_pop == {'a': 1, 'b': 2}
+
+ def test_dict_pop_missing_argument(self, custom_dict):
+ with pytest.raises(TypeError) as e:
+ dict_pop()
+ assert "dict_pop() missing 2 required positional arguments: 'dict_orig' and 'pop_keys'" in e.value.args[0]
+ with pytest.raises(TypeError) as e:
+ dict_pop(custom_dict)
+ assert "dict_pop() missing 1 required positional argument: 'pop_keys'" in e.value.args[0]
+
+
+class TestListPop:
+
+ @pytest.fixture
+ def custom_list(self):
+ return [1, 2, 3, 'a', 'bc']
+
+ def test_list_pop_single(self, custom_list):
+ l_pop = list_pop(custom_list, 1)
+ assert l_pop == [2, 3, 'a', 'bc']
+ l_pop = list_pop(custom_list, 'bc')
+ assert l_pop == [1, 2, 3, 'a']
+ l_pop = list_pop(custom_list, 5)
+ assert l_pop == custom_list
+
+ def test_list_pop_multiple(self, custom_list):
+ # all in list
+ l_pop = list_pop(custom_list, [2, 'a'])
+ assert l_pop == [1, 3, 'bc']
+ # one in one out
+ l_pop = list_pop(custom_list, ['bc', 10])
+ assert l_pop == [1, 2, 3, 'a']
+ # all out
+ l_pop = list_pop(custom_list, [10, 'aa'])
+ assert l_pop == custom_list
+
+ def test_list_pop_missing_argument(self, custom_list):
+ with pytest.raises(TypeError) as e:
+ list_pop()
+ assert "list_pop() missing 2 required positional arguments: 'list_full' and 'pop_items'" in e.value.args[0]
+ with pytest.raises(TypeError) as e:
+ list_pop(custom_list)
+ assert "list_pop() missing 1 required positional argument: 'pop_items'" in e.value.args[0]
+
+
+class TestLogger:
+
+ @pytest.fixture
+ def logger(self):
+ return Logger()
+
+ def test_init_default(self):
+ log = Logger()
+ assert log.formatter == "%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]"
+ assert log.log_file == Logger.setup_logging_path()
+ # assert PyTestRegex(
+ # ".*machinelearningtools/src/\.{2}/logging/logging_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.log") == log.log_file
+
+ def test_setup_logging_path_none(self):
+ log_file = Logger.setup_logging_path(None)
+ assert PyTestRegex(
+ ".*machinelearningtools/src/\.{2}/logging/logging_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.log") == log_file
+
+ @mock.patch("os.makedirs", side_effect=None)
+ def test_setup_logging_path_given(self, mock_makedirs):
+ path = "my/test/path"
+ log_path = Logger.setup_logging_path(path)
+ assert PyTestRegex("my/test/path/logging_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.log") == log_path
+
+ def test_logger_console_level0(self, logger):
+ consol = logger.logger_console(0)
+ assert isinstance(consol, logging.StreamHandler)
+ assert consol.level == 0
+ formatter = logging.Formatter(logger.formatter)
+ assert isinstance(formatter, logging.Formatter)
+
+ def test_logger_console_level1(self, logger):
+ consol = logger.logger_console(1)
+ assert isinstance(consol, logging.StreamHandler)
+ assert consol.level == 1
+ formatter = logging.Formatter(logger.formatter)
+ assert isinstance(formatter, logging.Formatter)
+
+ def test_logger_console_level_wrong_type(self, logger):
+ with pytest.raises(TypeError) as e:
+ logger.logger_console(1.5)
+ assert "Level not an integer or a valid string: 1.5" == e.value.args[0]
+
+
diff --git a/test/test_model_modules/test_advanced_paddings.py b/test/test_model_modules/test_advanced_paddings.py
index 5282eb6df34d4d395dbbdd1fd76fd71a95e9c8df..bbeaf1c745a63b3607062b0c4052088c9af06b92 100644
--- a/test/test_model_modules/test_advanced_paddings.py
+++ b/test/test_model_modules/test_advanced_paddings.py
@@ -417,3 +417,61 @@ class TestSymmerticPadding2D:
sym_pad = SymmetricPadding2D(padding=pad, name=layer_name)(input_x)
assert sym_pad.get_shape().as_list() == [None, 12, 10, 3]
assert sym_pad.name == 'SymPad_3x1/MirrorPad:0'
+
+
+class TestPadding2D:
+
+ @pytest.fixture
+ def input_x(self):
+ return keras.Input(shape=(32, 32, 3))
+
+ def test_init(self):
+ padding_layer = Padding2D('SymPad2D')
+ assert padding_layer.padding_type == 'SymPad2D'
+ assert padding_layer.allowed_paddings == {
+ 'RefPad2D': ReflectionPadding2D, 'ReflectionPadding2D': ReflectionPadding2D,
+ 'SymPad2D': SymmetricPadding2D, 'SymmetricPadding2D': SymmetricPadding2D,
+ 'ZeroPad2D': ZeroPadding2D, 'ZeroPadding2D': ZeroPadding2D
+ }
+
+
+ def test_check_and_get_padding_zero_padding(self):
+ assert Padding2D('ZeroPad2D')._check_and_get_padding() == ZeroPadding2D
+ assert Padding2D('ZeroPadding2D')._check_and_get_padding() == ZeroPadding2D
+ assert Padding2D(keras.layers.ZeroPadding2D)._check_and_get_padding() == ZeroPadding2D
+
+ def test_check_and_get_padding_sym_padding(self):
+ assert Padding2D('SymPad2D')._check_and_get_padding() == SymmetricPadding2D
+ assert Padding2D('SymmetricPadding2D')._check_and_get_padding() == SymmetricPadding2D
+ assert Padding2D(SymmetricPadding2D)._check_and_get_padding() == SymmetricPadding2D
+
+ def test_check_and_get_padding_ref_padding(self):
+ assert Padding2D('RefPad2D')._check_and_get_padding() == ReflectionPadding2D
+ assert Padding2D('ReflectionPadding2D')._check_and_get_padding() == ReflectionPadding2D
+ assert Padding2D(ReflectionPadding2D)._check_and_get_padding() == ReflectionPadding2D
+
+ def test_check_and_get_padding_raises(self,):
+ with pytest.raises(NotImplementedError) as einfo:
+ Padding2D('FalsePadding2D')._check_and_get_padding()
+ assert "`'FalsePadding2D'' is not implemented as padding. " \
+ "Use one of those: i) `RefPad2D', ii) `SymPad2D', iii) `ZeroPad2D'" in str(einfo.value)
+ with pytest.raises(TypeError) as einfo:
+ Padding2D(keras.layers.Conv2D)._check_and_get_padding()
+ assert "`Conv2D' is not a valid padding layer type. Use one of those: "\
+ "i) ReflectionPadding2D, ii) SymmetricPadding2D, iii) ZeroPadding2D" in str(einfo.value)
+
+ @pytest.mark.parametrize("pad_type", ["SymPad2D", "SymmetricPadding2D", SymmetricPadding2D,
+ "RefPad2D", "ReflectionPadding2D", ReflectionPadding2D,
+ "ZeroPad2D", "ZeroPadding2D", ZeroPadding2D])
+ def test_call(self, pad_type, input_x):
+ pd = Padding2D(pad_type)
+ if hasattr(pad_type, "__name__"):
+ layer_name = pad_type.__name__
+ else:
+ layer_name = pad_type
+ pd_ap = pd(padding=(1,2), name=f"{layer_name}_layer")(input_x)
+ assert pd_ap._keras_history[0].input_shape == (None, 32, 32, 3)
+ assert pd_ap._keras_history[0].output_shape == (None, 34, 36, 3)
+ assert pd_ap._keras_history[0].padding == ((1, 1), (2, 2))
+ assert pd_ap._keras_history[0].name == f"{layer_name}_layer"
+
diff --git a/test/test_model_modules/test_inception_model.py b/test/test_model_modules/test_inception_model.py
index 9dee30788c34cd8d1a7572947ea2e568ac2006b7..e5e92158425a73c5af1c6d1623d970e1037bbd80 100644
--- a/test/test_model_modules/test_inception_model.py
+++ b/test/test_model_modules/test_inception_model.py
@@ -277,48 +277,3 @@ class TestInceptionModelBase:
bn = base.batch_normalisation(input_x)._keras_history[0]
assert isinstance(bn, keras.layers.normalization.BatchNormalization)
assert bn.name == "Block_0a_BN"
-
- def test_padding_layer_zero_padding(self, base, input_x):
- padding_size = ((1, 1), (0, 0))
- zp = base.padding_layer('ZeroPad2D')
- assert zp == keras.layers.convolutional.ZeroPadding2D
- assert base.padding_layer('ZeroPadding2D') == keras.layers.convolutional.ZeroPadding2D
- assert base.padding_layer(keras.layers.ZeroPadding2D) == keras.layers.convolutional.ZeroPadding2D
- assert zp.__name__ == 'ZeroPadding2D'
- zp_ap = zp(padding=padding_size)(input_x)
- assert zp_ap._keras_history[0].padding == ((1, 1), (0, 0))
-
- def test_padding_layer_sym_padding(self, base, input_x):
- padding_size = ((1, 1), (0, 0))
- zp = base.padding_layer('SymPad2D')
- assert zp == SymmetricPadding2D
- assert base.padding_layer('SymmetricPadding2D') == SymmetricPadding2D
- assert base.padding_layer(SymmetricPadding2D) == SymmetricPadding2D
- assert zp.__name__ == 'SymmetricPadding2D'
- zp_ap = zp(padding=padding_size)(input_x)
- assert zp_ap._keras_history[0].padding == ((1, 1), (0, 0))
-
- def test_padding_layer_ref_padding(self, base, input_x):
- padding_size = ((1, 1), (0, 0))
- zp = base.padding_layer('RefPad2D')
- assert zp == ReflectionPadding2D
- assert base.padding_layer('ReflectionPadding2D') == ReflectionPadding2D
- assert base.padding_layer(ReflectionPadding2D) == ReflectionPadding2D
- assert zp.__name__ == 'ReflectionPadding2D'
- zp_ap = zp(padding=padding_size)(input_x)
- assert zp_ap._keras_history[0].padding == ((1, 1), (0, 0))
-
- def test_padding_layer_raises(self, base, input_x):
- with pytest.raises(NotImplementedError) as einfo:
- base.padding_layer('FalsePadding2D')
- assert "`'FalsePadding2D'' is not implemented as padding. " \
- "Use one of those: i) `RefPad2D', ii) `SymPad2D', iii) `ZeroPad2D'" in str(einfo.value)
- with pytest.raises(TypeError) as einfo:
- base.padding_layer(keras.layers.Conv2D)
- assert "`Conv2D' is not a valid padding layer type. Use one of those: "\
- "i) ReflectionPadding2D, ii) SymmetricPadding2D, iii) ZeroPadding2D" in str(einfo.value)
-
-
-
-
-
diff --git a/test/test_model_modules/test_model_class.py b/test/test_model_modules/test_model_class.py
index 0dbd2d9b67a0748bf09eb4f59e1888aae1ea405d..cee031749b193b91bd1cf16c02acfb3050eaed61 100644
--- a/test/test_model_modules/test_model_class.py
+++ b/test/test_model_modules/test_model_class.py
@@ -2,6 +2,18 @@ import keras
import pytest
from src.model_modules.model_class import AbstractModelClass
+from src.model_modules.model_class import MyPaperModel, MyTowerModel, MyLittleModel, MyBranchedModel
+
+
+class Paddings:
+ allowed_paddings = {"pad1": 34, "another_pad": True}
+
+
+class AbstractModelSubClass(AbstractModelClass):
+
+ def __init__(self):
+ super().__init__()
+ self.test_attr = "testAttr"
class TestAbstractModelClass:
@@ -10,9 +22,15 @@ class TestAbstractModelClass:
def amc(self):
return AbstractModelClass()
+ @pytest.fixture
+ def amsc(self):
+ return AbstractModelSubClass()
+
def test_init(self, amc):
assert amc.model is None
assert amc.loss is None
+ assert amc.model_name == "AbstractModelClass"
+ assert amc.custom_objects == {}
def test_model_property(self, amc):
amc.model = keras.Model()
@@ -27,3 +45,52 @@ class TestAbstractModelClass:
assert hasattr(amc, "compile") is True
assert hasattr(amc.model, "compile") is True
assert amc.compile == amc.model.compile
+
+ def test_get_settings(self, amc, amsc):
+ assert amc.get_settings() == {"model_name": "AbstractModelClass"}
+ assert amsc.get_settings() == {"test_attr": "testAttr", "model_name": "AbstractModelSubClass"}
+
+ def test_custom_objects(self, amc):
+ amc.custom_objects = {"Test": 123}
+ assert amc.custom_objects == {"Test": 123}
+
+ def test_set_custom_objects(self, amc):
+ amc.set_custom_objects(Test=22, minor_param="minor")
+ assert amc.custom_objects == {"Test": 22, "minor_param": "minor"}
+ amc.set_custom_objects(Test=2, minor_param1="minor1")
+ assert amc.custom_objects == {"Test": 2, "minor_param1": "minor1"}
+ paddings = Paddings()
+ amc.set_custom_objects(Test=1, Padding2D=paddings)
+ assert amc.custom_objects == {"Test": 1, "Padding2D": paddings, "pad1": 34, "another_pad": True}
+
+
+class TestMyPaperModel:
+
+ @pytest.fixture
+ def mpm(self):
+ return MyPaperModel(window_history_size=6, window_lead_time=4, channels=9)
+
+ def test_init(self, mpm):
+ # check if loss number of loss functions fit to model outputs
+ # same loss fkts. for all tails or different fkts. per tail
+ if isinstance(mpm.model.output_shape, list):
+ assert (callable(mpm.loss) or (len(mpm.loss) == 1)) or (len(mpm.loss) == len(mpm.model.output_shape))
+ elif isinstance(mpm.model.output_shape, tuple):
+ assert callable(mpm.loss) or (len(mpm.loss) == 1)
+
+ def test_set_model(self, mpm):
+ assert isinstance(mpm.model, keras.Model)
+ assert mpm.model.layers[0].output_shape == (None, 7, 1, 9)
+ # check output dimensions
+ if isinstance(mpm.model.output_shape, tuple):
+ assert mpm.model.output_shape == (None, 4)
+ elif isinstance(mpm.model.output_shape, list):
+ for tail_shape in mpm.model.output_shape:
+ assert tail_shape == (None, 4)
+ else:
+ raise TypeError(f"Type of model.output_shape as to be a tuple (one tail)"
+ f" or a list of tuples (multiple tails). Received: {type(mpm.model.output_shape)}")
+
+ def test_set_loss(self, mpm):
+ assert callable(mpm.loss) or (len(mpm.loss) > 0)
+
diff --git a/test/test_modules/test_experiment_setup.py b/test/test_modules/test_experiment_setup.py
index 894e4b552af4231ccc12fb85aaaebf5bbc23edf3..a3a83acf84e286d1f5da9b5caffa256fc0ca3327 100644
--- a/test/test_modules/test_experiment_setup.py
+++ b/test/test_modules/test_experiment_setup.py
@@ -85,12 +85,19 @@ class TestExperimentSetup:
# train parameters
assert data_store.get("start", "general.train") == "1997-01-01"
assert data_store.get("end", "general.train") == "2007-12-31"
+ assert data_store.get("min_length", "general.train") == 90
# validation parameters
assert data_store.get("start", "general.val") == "2008-01-01"
assert data_store.get("end", "general.val") == "2009-12-31"
+ assert data_store.get("min_length", "general.val") == 90
# test parameters
assert data_store.get("start", "general.test") == "2010-01-01"
assert data_store.get("end", "general.test") == "2017-12-31"
+ assert data_store.get("min_length", "general.test") == 90
+ # train_val parameters
+ assert data_store.get("start", "general.train_val") == "1997-01-01"
+ assert data_store.get("end", "general.train_val") == "2009-12-31"
+ assert data_store.get("min_length", "general.train_val") == 180
# use all stations on all data sets (train, val, test)
assert data_store.get("use_all_stations_on_all_data_sets", "general") is True
@@ -104,7 +111,7 @@ class TestExperimentSetup:
interpolate_dim="int_dim", interpolate_method="cubic", limit_nan_fill=5, train_start="2000-01-01",
train_end="2000-01-02", val_start="2000-01-03", val_end="2000-01-04", test_start="2000-01-05",
test_end="2000-01-06", use_all_stations_on_all_data_sets=False, trainable=False,
- fraction_of_train=0.5, experiment_path=experiment_path, create_new_model=True)
+ fraction_of_train=0.5, experiment_path=experiment_path, create_new_model=True, val_min_length=20)
exp_setup = ExperimentSetup(**kwargs)
data_store = exp_setup.data_store
# experiment setup
@@ -139,12 +146,19 @@ class TestExperimentSetup:
# train parameters
assert data_store.get("start", "general.train") == "2000-01-01"
assert data_store.get("end", "general.train") == "2000-01-02"
+ assert data_store.get("min_length", "general.train") == 90
# validation parameters
assert data_store.get("start", "general.val") == "2000-01-03"
assert data_store.get("end", "general.val") == "2000-01-04"
+ assert data_store.get("min_length", "general.val") == 20
# test parameters
assert data_store.get("start", "general.test") == "2000-01-05"
assert data_store.get("end", "general.test") == "2000-01-06"
+ assert data_store.get("min_length", "general.test") == 90
+ # train_val parameters
+ assert data_store.get("start", "general.train_val") == "2000-01-01"
+ assert data_store.get("end", "general.train_val") == "2000-01-04"
+ assert data_store.get("min_length", "general.train_val") == 110
# use all stations on all data sets (train, val, test)
assert data_store.get("use_all_stations_on_all_data_sets", "general.test") is False
diff --git a/test/test_modules/test_model_setup.py b/test/test_modules/test_model_setup.py
index ade35a244601d138d22af6305e67b5aeae964680..9ff7494ff0540c9c96c1343b4f44fece08bfe4ce 100644
--- a/test/test_modules/test_model_setup.py
+++ b/test/test_modules/test_model_setup.py
@@ -4,6 +4,7 @@ import pytest
from src.data_handling.data_generator import DataGenerator
from src.datastore import EmptyScope
+from src.model_modules.keras_extensions import CallbackHandler
from src.model_modules.model_class import AbstractModelClass
from src.run_modules.model_setup import ModelSetup
from src.run_modules.run_environment import RunEnvironment
@@ -61,6 +62,18 @@ class TestModelSetup:
setup.checkpoint_name = "TestName"
setup._set_callbacks()
assert "general.modeltest" in setup.data_store.search_name("callbacks")
+ callbacks = setup.data_store.get("callbacks", "general.modeltest")
+ assert len(callbacks.get_callbacks()) == 3
+
+ def test_set_callbacks_no_lr_decay(self, setup):
+ setup.data_store.set("lr_decay", None, "general.model")
+ assert "general.modeltest" not in setup.data_store.search_name("callbacks")
+ setup.checkpoint_name = "TestName"
+ setup._set_callbacks()
+ callbacks: CallbackHandler = setup.data_store.get("callbacks", "general.modeltest")
+ assert len(callbacks.get_callbacks()) == 2
+ with pytest.raises(IndexError):
+ callbacks.get_callback_by_name("lr_decay")
def test_get_model_settings(self, setup_with_model):
with pytest.raises(EmptyScope):
@@ -73,7 +86,7 @@ class TestModelSetup:
setup_with_gen.build_model()
assert isinstance(setup_with_gen.model, AbstractModelClass)
expected = {"window_history_size", "window_lead_time", "channels", "dropout_rate", "regularizer", "initial_lr",
- "optimizer", "lr_decay", "epochs", "batch_size", "activation"}
+ "optimizer", "epochs", "batch_size", "activation"}
assert expected <= self.current_scope_as_set(setup_with_gen)
def test_set_channels(self, setup_with_gen_tiny):
diff --git a/test/test_modules/test_pre_processing.py b/test/test_modules/test_pre_processing.py
index d58cbd41e2ce4f25f4cd79127256e313b4aac649..b29ed1e21480a869e4c118332c18b6edd8ac23a5 100644
--- a/test/test_modules/test_pre_processing.py
+++ b/test/test_modules/test_pre_processing.py
@@ -36,10 +36,11 @@ class TestPreProcessing:
def test_init(self, caplog):
ExperimentSetup(parser_args={}, stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087'],
statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'})
+ caplog.clear()
caplog.set_level(logging.INFO)
with PreProcessing():
assert caplog.record_tuples[0] == ('root', 20, 'PreProcessing started')
- assert caplog.record_tuples[1] == ('root', 20, 'check valid stations started')
+ assert caplog.record_tuples[1] == ('root', 20, 'check valid stations started (all)')
assert caplog.record_tuples[-1] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 5 '
r'station\(s\). Found 5/5 valid stations.'))
RunEnvironment().__del__()
@@ -54,7 +55,8 @@ class TestPreProcessing:
assert obj_with_exp_setup.data_store.search_name("generator") == []
obj_with_exp_setup.split_train_val_test()
data_store = obj_with_exp_setup.data_store
- expected_params = ["generator", "start", "end", "stations", "permute_data"]
+ expected_params = ["generator", "start", "end", "stations", "permute_data", "min_length", "extreme_values",
+ "extremes_on_right_tail_only", "upsampling"]
assert data_store.search_scope("general.train") == sorted(expected_params)
assert data_store.search_name("generator") == sorted(["general.train", "general.val", "general.test",
"general.train_val"])
@@ -81,16 +83,18 @@ class TestPreProcessing:
data_store.get("generator", "general")
assert data_store.get("stations", "general.awesome") == ['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087']
- def test_check_valid_stations(self, caplog, obj_with_exp_setup):
+ @pytest.mark.parametrize("name", (None, "tester"))
+ def test_check_valid_stations(self, caplog, obj_with_exp_setup, name):
pre = obj_with_exp_setup
caplog.set_level(logging.INFO)
args = pre.data_store.create_args_dict(DEFAULT_ARGS_LIST)
kwargs = pre.data_store.create_args_dict(DEFAULT_KWARGS_LIST)
stations = pre.data_store.get("stations", "general")
- valid_stations = pre.check_valid_stations(args, kwargs, stations)
+ valid_stations = pre.check_valid_stations(args, kwargs, stations, name=name)
assert len(valid_stations) < len(stations)
assert valid_stations == stations[:-1]
- assert caplog.record_tuples[0] == ('root', 20, 'check valid stations started')
+ expected = 'check valid stations started (tester)' if name else 'check valid stations started'
+ assert caplog.record_tuples[0] == ('root', 20, expected)
assert caplog.record_tuples[-1] == ('root', 20, PyTestRegex(r'run for \d+:\d+:\d+ \(hh:mm:ss\) to check 6 '
r'station\(s\). Found 5/6 valid stations.'))