diff --git a/.gitignore b/.gitignore
index e109cec7c7622cee9d9a635cc458b9c662fc4761..9ac8bb7635bb12d5e3bc32182a90d0f3ba985c58 100644
--- a/.gitignore
+++ b/.gitignore
@@ -73,7 +73,7 @@ report.html
# secret variables #
####################
-/src/configuration/join_settings.py
+/mlair/configuration/join_settings.py
# ignore locally build documentation #
######################################
diff --git a/README.md b/README.md
index baae0af91036da10ba70f154ac875c18908858c3..7696415b9d9ad2168ad54b2d45b2b1606d39d89f 100644
--- a/README.md
+++ b/README.md
@@ -1,22 +1,25 @@
-# MachineLearningTools
-
-This is a collection of all relevant functions used for ML stuff in the ESDE group
-
-## Inception Model
-
-See a description [here](https://towardsdatascience.com/a-simple-guide-to-the-versions-of-the-inception-network-7fc52b863202)
-or take a look on the papers [Going Deeper with Convolutions (Szegedy et al., 2014)](https://arxiv.org/abs/1409.4842)
-and [Network In Network (Lin et al., 2014)](https://arxiv.org/abs/1312.4400).
+# MLAir - Machine Learning on Air Data
+MLAir (Machine Learning on Air data) is an environment that simplifies and accelerates the creation of new machine
+learning (ML) models for the analysis and forecasting of meteorological and air quality time series.
# Installation
* Install __proj__ on your machine using the console. E.g. for opensuse / leap `zypper install proj`
-* c++ compiler required for cartopy installation
-
-## HPC - JUWELS and HDFML setup
-The following instruction guide you throug the installation on JUWELS and HDFML.
-* Clone the repo to HPC system (we recommend to place it in `/p/projects/<project name>`.
+* A c++ compiler is required for the installation of the program __cartopy__
+* Install all requirements from `requirements.txt` preferably in a virtual environment
+* Installation of MLAir:
+ * Either clone MLAir from its repository in gitlab (link??) and use it without installation
+ * or download the distribution file (?? .whl) and install it via `pip install <??>`. In this case, you can simply
+ import MLAir in any python script inside your virtual environment using `import mlair`.
+
+## Special instructions for installation on Jülich HPC systems
+
+_Please note, that the HPC setup is customised for JUWELS and HDFML. When using another HPC system, you can use the HPC
+setup files as a skeleton and customise it to your needs._
+
+The following instruction guide you through the installation on JUWELS and HDFML.
+* Clone the repo to HPC system (we recommend to place it in `/p/projects/<project name>`).
* Setup venv by executing `source setupHPC.sh`. This script loads all pre-installed modules and creates a venv for
all other packages. Furthermore, it creates slurm/batch scripts to execute code on compute nodes. <br>
You have to enter the HPC project's budget name (--account flag).
@@ -27,9 +30,6 @@ You have to enter the HPC project's budget name (--account flag).
* Execute either `sbatch run_juwels_develgpus.bash` or `sbatch run_hdfml_batch.bash` to verify that the setup went well.
* Currently cartopy is not working on our HPC system, therefore PlotStations does not create any output.
-### HPC JUWELS and HDFML remarks
-Please note, that the HPC setup is customised for JUWELS and HDFML. When using another HPC system, you can use the HPC setup files as a skeleton and customise it to your needs.
-
Note: The method `PartitionCheck` currently only checks if the hostname starts with `ju` or `hdfmll`.
Therefore, it might be necessary to adopt the `if` statement in `PartitionCheck._run`.
@@ -39,8 +39,7 @@ Therefore, it might be necessary to adopt the `if` statement in `PartitionCheck.
* To use hourly data from ToarDB via JOIN interface, a private token is required. Request your personal access token and
add it to `src/join_settings.py` in the hourly data section. Replace the `TOAR_SERVICE_URL` and the `Authorization`
value. To make sure, that this **sensitive** data is not uploaded to the remote server, use the following command to
-prevent git from tracking this file: `git update-index --assume-unchanged src/join_settings.py
-`
+prevent git from tracking this file: `git update-index --assume-unchanged src/join_settings.py`
# Customise your experiment
@@ -97,3 +96,10 @@ station-wise std is a decent estimate of the true std.
scaling values instead of the calculation method. For method *centre*, std can still be None, but is required for the
*standardise* method. **Important**: Format of given values **must** match internal data format of DataPreparation
class: `xr.DataArray` with `dims=["variables"]` and one value for each variable.
+
+
+## Inception Model
+
+See a description [here](https://towardsdatascience.com/a-simple-guide-to-the-versions-of-the-inception-network-7fc52b863202)
+or take a look on the papers [Going Deeper with Convolutions (Szegedy et al., 2014)](https://arxiv.org/abs/1409.4842)
+and [Network In Network (Lin et al., 2014)](https://arxiv.org/abs/1312.4400).
diff --git a/docs/_source/conf.py b/docs/_source/conf.py
index 6363f57eb45e686f6f2ef8ab07806e4feba0fe2d..ac1131a008f5c95a62718def6046085294f6efba 100644
--- a/docs/_source/conf.py
+++ b/docs/_source/conf.py
@@ -55,7 +55,7 @@ extensions = [
autosummary_generate = True
autoapi_type = 'python'
-autoapi_dirs = ['../../src/.']
+autoapi_dirs = ['../../mlair/.']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
diff --git a/src/__init__.py b/mlair/__init__.py
similarity index 79%
rename from src/__init__.py
rename to mlair/__init__.py
index 5b7073ff042f6173fd78362f55d698eb6745552f..8ff82e6c3457922927fa637579935a172a64bb7b 100644
--- a/src/__init__.py
+++ b/mlair/__init__.py
@@ -4,9 +4,9 @@ __version_info__ = {
'micro': 0,
}
-from src.run_modules import *
-from src.workflows import DefaultWorkflow, Workflow
-
+from mlair.run_modules import *
+from mlair.workflows import DefaultWorkflow, Workflow
+from mlair.run_script import run
def get_version():
diff --git a/src/configuration/.gitignore b/mlair/configuration/.gitignore
similarity index 100%
rename from src/configuration/.gitignore
rename to mlair/configuration/.gitignore
diff --git a/src/configuration/__init__.py b/mlair/configuration/__init__.py
similarity index 100%
rename from src/configuration/__init__.py
rename to mlair/configuration/__init__.py
diff --git a/src/configuration/defaults.py b/mlair/configuration/defaults.py
similarity index 100%
rename from src/configuration/defaults.py
rename to mlair/configuration/defaults.py
diff --git a/src/configuration/join_settings.py b/mlair/configuration/join_settings.py
similarity index 100%
rename from src/configuration/join_settings.py
rename to mlair/configuration/join_settings.py
diff --git a/src/configuration/path_config.py b/mlair/configuration/path_config.py
similarity index 68%
rename from src/configuration/path_config.py
rename to mlair/configuration/path_config.py
index 7af25875eea58de081012fc6040a76a04f001d54..0ef082b58cf7028ea4f71e86b6d0c4ecad6ff54d 100644
--- a/src/configuration/path_config.py
+++ b/mlair/configuration/path_config.py
@@ -6,7 +6,8 @@ import re
import socket
from typing import Tuple
-ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+# ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+ROOT_PATH = os.getcwd()
def prepare_host(create_new=True, data_path=None, sampling="daily") -> str:
@@ -23,35 +24,38 @@ def prepare_host(create_new=True, data_path=None, sampling="daily") -> str:
:return: full path to data
"""
- hostname = get_host()
- user = getpass.getuser()
- runner_regex = re.compile(r"runner-.*-project-2411-concurrent-\d+")
- if hostname == "ZAM144":
- path = f"/home/{user}/Data/toar_{sampling}/"
- elif hostname == "zam347":
- path = f"/home/{user}/Data/toar_{sampling}/"
- elif hostname == "linux-aa9b":
- path = f"/home/{user}/machinelearningtools/data/toar_{sampling}/"
- elif (len(hostname) > 2) and (hostname[:2] == "jr"):
- path = f"/p/project/cjjsc42/{user}/DATA/toar_{sampling}/"
- elif (len(hostname) > 2) and (hostname[:2] in ['jw', 'ju'] or hostname[:5] in ['hdfml']):
- path = f"/p/project/deepacf/intelliaq/{user}/DATA/toar_{sampling}/"
- elif runner_regex.match(hostname) is not None:
- path = f"/home/{user}/machinelearningtools/data/toar_{sampling}/"
- else:
- raise OSError(f"unknown host '{hostname}'")
- if not os.path.exists(path):
+ if data_path is None:
+ hostname = get_host()
+ user = getpass.getuser()
+ runner_regex = re.compile(r"runner-.*-project-2411-concurrent-\d+")
+ if hostname == "ZAM144":
+ data_path = f"/home/{user}/Data/toar_{sampling}/"
+ elif hostname == "zam347":
+ data_path = f"/home/{user}/Data/toar_{sampling}/"
+ elif hostname == "linux-aa9b":
+ data_path = f"/home/{user}/machinelearningtools/data/toar_{sampling}/"
+ elif (len(hostname) > 2) and (hostname[:2] == "jr"):
+ data_path = f"/p/project/cjjsc42/{user}/DATA/toar_{sampling}/"
+ elif (len(hostname) > 2) and (hostname[:2] in ['jw', 'ju'] or hostname[:5] in ['hdfml']):
+ data_path = f"/p/project/deepacf/intelliaq/{user}/DATA/toar_{sampling}/"
+ elif runner_regex.match(hostname) is not None:
+ data_path = f"/home/{user}/machinelearningtools/data/toar_{sampling}/"
+ else:
+ data_path = os.path.join(os.getcwd(), "data", sampling)
+ # raise OSError(f"unknown host '{hostname}'")
+
+ if not os.path.exists(data_path):
try:
if create_new:
- check_path_and_create(path)
- return path
+ check_path_and_create(data_path)
+ return data_path
else:
raise PermissionError
except PermissionError:
- raise NotADirectoryError(f"path '{path}' does not exist for host '{hostname}'.")
+ raise NotADirectoryError(f"path '{data_path}' does not exist for host '{hostname}'.")
else:
- logging.debug(f"set path to: {path}")
- return path
+ logging.debug(f"set path to: {data_path}")
+ return data_path
def set_experiment_path(name: str, path: str = None) -> str:
diff --git a/src/data_handling/__init__.py b/mlair/data_handling/__init__.py
similarity index 100%
rename from src/data_handling/__init__.py
rename to mlair/data_handling/__init__.py
diff --git a/src/data_handling/bootstraps.py b/mlair/data_handling/bootstraps.py
similarity index 98%
rename from src/data_handling/bootstraps.py
rename to mlair/data_handling/bootstraps.py
index f50775900c053cbef0c94e6a3e2743c9a017bf88..4e72b2b81476d04aec819cc6be0fdfd585e5eaf9 100644
--- a/src/data_handling/bootstraps.py
+++ b/mlair/data_handling/bootstraps.py
@@ -22,8 +22,8 @@ import keras
import numpy as np
import xarray as xr
-from src import helpers
-from src.data_handling.data_generator import DataGenerator
+from mlair import helpers
+from mlair.data_handling.data_generator import DataGenerator
class BootStrapGenerator(keras.utils.Sequence):
@@ -361,9 +361,9 @@ class BootStraps:
if __name__ == "__main__":
- from src.run_modules.experiment_setup import ExperimentSetup
- from src.run_modules.run_environment import RunEnvironment
- from src.run_modules.pre_processing import PreProcessing
+ from mlair.run_modules.experiment_setup import ExperimentSetup
+ from mlair.run_modules.run_environment import RunEnvironment
+ from mlair.run_modules.pre_processing import PreProcessing
formatter = '%(asctime)s - %(levelname)s: %(message)s [%(filename)s:%(funcName)s:%(lineno)s]'
logging.basicConfig(format=formatter, level=logging.INFO)
diff --git a/src/data_handling/data_distributor.py b/mlair/data_handling/data_distributor.py
similarity index 98%
rename from src/data_handling/data_distributor.py
rename to mlair/data_handling/data_distributor.py
index 2600afcbd8948c26a2b4cf37329b424cac69f40a..bba5f2636f802e2d6843ef4a5ba5e6537c70dd61 100644
--- a/src/data_handling/data_distributor.py
+++ b/mlair/data_handling/data_distributor.py
@@ -29,7 +29,7 @@ import math
import keras
import numpy as np
-from src.data_handling.data_generator import DataGenerator
+from mlair.data_handling.data_generator import DataGenerator
class Distributor(keras.utils.Sequence):
diff --git a/src/data_handling/data_generator.py b/mlair/data_handling/data_generator.py
similarity index 99%
rename from src/data_handling/data_generator.py
rename to mlair/data_handling/data_generator.py
index 6747e82e0da2d1a68c99a09d75c76cdcd53a05ba..4c61454b0c751eeb6338f6f4e9072cabe14379a1 100644
--- a/src/data_handling/data_generator.py
+++ b/mlair/data_handling/data_generator.py
@@ -12,9 +12,9 @@ import dask.array as da
import keras
import xarray as xr
-from src import helpers
-from src.data_handling.data_preparation import DataPrep
-from src.helpers.join import EmptyQueryResult
+from mlair import helpers
+from mlair.data_handling.data_preparation import DataPrep
+from mlair.helpers.join import EmptyQueryResult
number = Union[float, int]
num_or_list = Union[number, List[number]]
diff --git a/src/data_handling/data_preparation.py b/mlair/data_handling/data_preparation.py
similarity index 99%
rename from src/data_handling/data_preparation.py
rename to mlair/data_handling/data_preparation.py
index fa7388e7d5510eae71a65ebc108f9a6dc3b0a2ff..f500adec5c7a2d7fac67f3e6e9ba2fc61079c115 100644
--- a/src/data_handling/data_preparation.py
+++ b/mlair/data_handling/data_preparation.py
@@ -13,9 +13,9 @@ import numpy as np
import pandas as pd
import xarray as xr
-from src.configuration import check_path_and_create
-from src import helpers
-from src.helpers import join, statistics
+from mlair.configuration import check_path_and_create
+from mlair import helpers
+from mlair.helpers import join, statistics
# define a more general date type for type hinting
date = Union[dt.date, dt.datetime]
diff --git a/src/helpers/__init__.py b/mlair/helpers/__init__.py
similarity index 100%
rename from src/helpers/__init__.py
rename to mlair/helpers/__init__.py
diff --git a/src/helpers/datastore.py b/mlair/helpers/datastore.py
similarity index 100%
rename from src/helpers/datastore.py
rename to mlair/helpers/datastore.py
diff --git a/src/helpers/helpers.py b/mlair/helpers/helpers.py
similarity index 100%
rename from src/helpers/helpers.py
rename to mlair/helpers/helpers.py
diff --git a/src/helpers/join.py b/mlair/helpers/join.py
similarity index 98%
rename from src/helpers/join.py
rename to mlair/helpers/join.py
index 7d9c3aad23c402ae63f26bdf998074a86e35ffbf..a3c6876e3ea43ff4d03243430cf6cd791d62dec2 100644
--- a/src/helpers/join.py
+++ b/mlair/helpers/join.py
@@ -9,8 +9,8 @@ from typing import Iterator, Union, List, Dict
import pandas as pd
import requests
-from src import helpers
-from src.configuration.join_settings import join_settings
+from mlair import helpers
+from mlair.configuration.join_settings import join_settings
# join_url_base = 'https://join.fz-juelich.de/services/rest/surfacedata/'
str_or_none = Union[str, None]
diff --git a/src/helpers/logger.py b/mlair/helpers/logger.py
similarity index 100%
rename from src/helpers/logger.py
rename to mlair/helpers/logger.py
diff --git a/src/helpers/statistics.py b/mlair/helpers/statistics.py
similarity index 100%
rename from src/helpers/statistics.py
rename to mlair/helpers/statistics.py
diff --git a/src/helpers/testing.py b/mlair/helpers/testing.py
similarity index 100%
rename from src/helpers/testing.py
rename to mlair/helpers/testing.py
diff --git a/src/helpers/time_tracking.py b/mlair/helpers/time_tracking.py
similarity index 100%
rename from src/helpers/time_tracking.py
rename to mlair/helpers/time_tracking.py
diff --git a/src/model_modules/__init__.py b/mlair/model_modules/__init__.py
similarity index 100%
rename from src/model_modules/__init__.py
rename to mlair/model_modules/__init__.py
diff --git a/src/model_modules/advanced_paddings.py b/mlair/model_modules/advanced_paddings.py
similarity index 100%
rename from src/model_modules/advanced_paddings.py
rename to mlair/model_modules/advanced_paddings.py
diff --git a/src/model_modules/flatten.py b/mlair/model_modules/flatten.py
similarity index 100%
rename from src/model_modules/flatten.py
rename to mlair/model_modules/flatten.py
diff --git a/src/model_modules/inception_model.py b/mlair/model_modules/inception_model.py
similarity index 99%
rename from src/model_modules/inception_model.py
rename to mlair/model_modules/inception_model.py
index 74cd4d806f706a70d554adae468e7fa8c5de153e..d7354c37899bbb7d8f80bc76b4cd9237c7df96dc 100644
--- a/src/model_modules/inception_model.py
+++ b/mlair/model_modules/inception_model.py
@@ -6,7 +6,7 @@ import logging
import keras
import keras.layers as layers
-from src.model_modules.advanced_paddings import PadUtils, ReflectionPadding2D, Padding2D
+from mlair.model_modules.advanced_paddings import PadUtils, ReflectionPadding2D, Padding2D
class InceptionModelBase:
diff --git a/src/model_modules/keras_extensions.py b/mlair/model_modules/keras_extensions.py
similarity index 99%
rename from src/model_modules/keras_extensions.py
rename to mlair/model_modules/keras_extensions.py
index 479913811a668d8330a389b2876360f096f57dbf..33358e566ef80f28ee7740531b71d1a83abde115 100644
--- a/src/model_modules/keras_extensions.py
+++ b/mlair/model_modules/keras_extensions.py
@@ -13,7 +13,7 @@ import numpy as np
from keras import backend as K
from keras.callbacks import History, ModelCheckpoint, Callback
-from src import helpers
+from mlair import helpers
class HistoryAdvanced(History):
diff --git a/src/model_modules/linear_model.py b/mlair/model_modules/linear_model.py
similarity index 100%
rename from src/model_modules/linear_model.py
rename to mlair/model_modules/linear_model.py
diff --git a/src/model_modules/loss.py b/mlair/model_modules/loss.py
similarity index 100%
rename from src/model_modules/loss.py
rename to mlair/model_modules/loss.py
diff --git a/src/model_modules/model_class.py b/mlair/model_modules/model_class.py
similarity index 97%
rename from src/model_modules/model_class.py
rename to mlair/model_modules/model_class.py
index dab2e168c5a9f87d4aee42fc94489fd0fa67772a..dbf7771e376d9bd68f594ecad319554c984dc974 100644
--- a/src/model_modules/model_class.py
+++ b/mlair/model_modules/model_class.py
@@ -108,7 +108,7 @@ True
"""
-import src.model_modules.keras_extensions
+import mlair.model_modules.keras_extensions
__author__ = "Lukas Leufen, Felix Kleinert"
# __date__ = '2019-12-12'
@@ -119,9 +119,9 @@ from typing import Any, Callable, Dict
import keras
import tensorflow as tf
-from src.model_modules.inception_model import InceptionModelBase
-from src.model_modules.flatten import flatten_tail
-from src.model_modules.advanced_paddings import PadUtils, Padding2D
+from mlair.model_modules.inception_model import InceptionModelBase
+from mlair.model_modules.flatten import flatten_tail
+from mlair.model_modules.advanced_paddings import PadUtils, Padding2D
class AbstractModelClass(ABC):
@@ -392,8 +392,8 @@ class MyLittleModel(AbstractModelClass):
def set_compile_options(self):
self.initial_lr = 1e-2
self.optimizer = keras.optimizers.SGD(lr=self.initial_lr, momentum=0.9)
- self.lr_decay = src.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94,
- epochs_drop=10)
+ self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94,
+ epochs_drop=10)
self.compile_options = {"loss": keras.losses.mean_squared_error, "metrics": ["mse", "mae"]}
@@ -468,8 +468,8 @@ class MyBranchedModel(AbstractModelClass):
def set_compile_options(self):
self.initial_lr = 1e-2
self.optimizer = keras.optimizers.SGD(lr=self.initial_lr, momentum=0.9)
- self.lr_decay = src.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94,
- epochs_drop=10)
+ self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94,
+ epochs_drop=10)
self.compile_options = {"loss": [keras.losses.mean_absolute_error] + [keras.losses.mean_squared_error] + [
keras.losses.mean_squared_error], "metrics": ["mse", "mae"]}
@@ -497,8 +497,8 @@ class MyTowerModel(AbstractModelClass):
self.dropout_rate = 1e-2
self.regularizer = keras.regularizers.l2(0.1)
self.initial_lr = 1e-2
- self.lr_decay = src.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94,
- epochs_drop=10)
+ self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94,
+ epochs_drop=10)
self.activation = keras.layers.PReLU
# apply to model
@@ -609,8 +609,8 @@ class MyPaperModel(AbstractModelClass):
self.dropout_rate = .3
self.regularizer = keras.regularizers.l2(0.001)
self.initial_lr = 1e-3
- self.lr_decay = src.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94,
- epochs_drop=10)
+ self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr, drop=.94,
+ epochs_drop=10)
self.activation = keras.layers.ELU
self.padding = "SymPad2D"
diff --git a/src/plotting/__init__.py b/mlair/plotting/__init__.py
similarity index 100%
rename from src/plotting/__init__.py
rename to mlair/plotting/__init__.py
diff --git a/src/plotting/postprocessing_plotting.py b/mlair/plotting/postprocessing_plotting.py
similarity index 99%
rename from src/plotting/postprocessing_plotting.py
rename to mlair/plotting/postprocessing_plotting.py
index 4b7f15219ee5506f34e4fc2d76c15fb0e569394d..cf96948035d4e6da497c2f9502cdb467ee3ba9c0 100644
--- a/src/plotting/postprocessing_plotting.py
+++ b/mlair/plotting/postprocessing_plotting.py
@@ -18,9 +18,9 @@ import seaborn as sns
import xarray as xr
from matplotlib.backends.backend_pdf import PdfPages
-from src import helpers
-from src.data_handling import DataGenerator
-from src.helpers import TimeTrackingWrapper
+from mlair import helpers
+from mlair.data_handling import DataGenerator
+from mlair.helpers import TimeTrackingWrapper
logging.getLogger('matplotlib').setLevel(logging.WARNING)
diff --git a/src/plotting/tracker_plot.py b/mlair/plotting/tracker_plot.py
similarity index 99%
rename from src/plotting/tracker_plot.py
rename to mlair/plotting/tracker_plot.py
index 20db5d9d9f22df548b1d499c4e8e0faa3fbfa1ee..406c32feb1ebda2d32d886051e32778d6c17f5db 100644
--- a/src/plotting/tracker_plot.py
+++ b/mlair/plotting/tracker_plot.py
@@ -4,7 +4,7 @@ import numpy as np
import os
from typing import Union, List, Optional, Dict
-from src.helpers import to_list
+from mlair.helpers import to_list
from matplotlib import pyplot as plt, lines as mlines, ticker as ticker
from matplotlib.patches import Rectangle
diff --git a/src/plotting/training_monitoring.py b/mlair/plotting/training_monitoring.py
similarity index 98%
rename from src/plotting/training_monitoring.py
rename to mlair/plotting/training_monitoring.py
index 473b966ce52ee7e2885bc14beef2e68b8835b15e..913c11dd8a4e0d23c2bde6864c12f17c65922644 100644
--- a/src/plotting/training_monitoring.py
+++ b/mlair/plotting/training_monitoring.py
@@ -10,7 +10,7 @@ import matplotlib
import matplotlib.pyplot as plt
import pandas as pd
-from src.model_modules.keras_extensions import LearningRateDecay
+from mlair.model_modules.keras_extensions import LearningRateDecay
matplotlib.use('Agg')
history_object = Union[Dict, keras.callbacks.History]
diff --git a/src/run_modules/README.md b/mlair/run_modules/README.md
similarity index 100%
rename from src/run_modules/README.md
rename to mlair/run_modules/README.md
diff --git a/mlair/run_modules/__init__.py b/mlair/run_modules/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ba38d3e90fb5d66c4129f6645ef34b8137e48375
--- /dev/null
+++ b/mlair/run_modules/__init__.py
@@ -0,0 +1,7 @@
+from mlair.run_modules.experiment_setup import ExperimentSetup
+from mlair.run_modules.model_setup import ModelSetup
+from mlair.run_modules.partition_check import PartitionCheck
+from mlair.run_modules.post_processing import PostProcessing
+from mlair.run_modules.pre_processing import PreProcessing
+from mlair.run_modules.run_environment import RunEnvironment
+from mlair.run_modules.training import Training
diff --git a/src/run_modules/experiment_setup.py b/mlair/run_modules/experiment_setup.py
similarity index 98%
rename from src/run_modules/experiment_setup.py
rename to mlair/run_modules/experiment_setup.py
index a93fe403e11b6483832df417b0e97462bc252a4a..c40847eddc054e3fc32923d5fee87d4458f76754 100644
--- a/src/run_modules/experiment_setup.py
+++ b/mlair/run_modules/experiment_setup.py
@@ -6,9 +6,9 @@ import logging
import os
from typing import Union, Dict, Any, List
-from src.configuration import path_config
-from src import helpers
-from src.configuration.defaults import DEFAULT_STATIONS, DEFAULT_VAR_ALL_DICT, DEFAULT_NETWORK, DEFAULT_STATION_TYPE, \
+from mlair.configuration import path_config
+from mlair import helpers
+from mlair.configuration.defaults import DEFAULT_STATIONS, DEFAULT_VAR_ALL_DICT, DEFAULT_NETWORK, DEFAULT_STATION_TYPE, \
DEFAULT_START, DEFAULT_END, DEFAULT_WINDOW_HISTORY_SIZE, DEFAULT_OVERWRITE_LOCAL_DATA, DEFAULT_TRANSFORMATION, \
DEFAULT_HPC_LOGIN_LIST, DEFAULT_HPC_HOST_LIST, DEFAULT_CREATE_NEW_MODEL, DEFAULT_TRAINABLE, \
DEFAULT_FRACTION_OF_TRAINING, DEFAULT_EXTREME_VALUES, DEFAULT_EXTREMES_ON_RIGHT_TAIL_ONLY, DEFAULT_PERMUTE_DATA, \
@@ -18,8 +18,8 @@ from src.configuration.defaults import DEFAULT_STATIONS, DEFAULT_VAR_ALL_DICT, D
DEFAULT_VAL_MIN_LENGTH, DEFAULT_TEST_START, DEFAULT_TEST_END, DEFAULT_TEST_MIN_LENGTH, DEFAULT_TRAIN_VAL_MIN_LENGTH, \
DEFAULT_USE_ALL_STATIONS_ON_ALL_DATA_SETS, DEFAULT_EVALUATE_BOOTSTRAPS, DEFAULT_CREATE_NEW_BOOTSTRAPS, \
DEFAULT_NUMBER_OF_BOOTSTRAPS, DEFAULT_PLOT_LIST
-from src.run_modules.run_environment import RunEnvironment
-from src.model_modules.model_class import MyLittleModel as VanillaModel
+from mlair.run_modules.run_environment import RunEnvironment
+from mlair.model_modules.model_class import MyLittleModel as VanillaModel
class ExperimentSetup(RunEnvironment):
diff --git a/src/run_modules/model_setup.py b/mlair/run_modules/model_setup.py
similarity index 97%
rename from src/run_modules/model_setup.py
rename to mlair/run_modules/model_setup.py
index f9683b953d85bacf6e452e0a1922e85dfe946cd1..6c05a5731056cd3926e319b6d549ca3f64ed3873 100644
--- a/src/run_modules/model_setup.py
+++ b/mlair/run_modules/model_setup.py
@@ -9,8 +9,8 @@ import os
import keras
import tensorflow as tf
-from src.model_modules.keras_extensions import HistoryAdvanced, CallbackHandler
-from src.run_modules.run_environment import RunEnvironment
+from mlair.model_modules.keras_extensions import HistoryAdvanced, CallbackHandler
+from mlair.run_modules.run_environment import RunEnvironment
class ModelSetup(RunEnvironment):
diff --git a/src/run_modules/partition_check.py b/mlair/run_modules/partition_check.py
similarity index 93%
rename from src/run_modules/partition_check.py
rename to mlair/run_modules/partition_check.py
index 8f4c703e6b94f11905121d93c44dd8bf583abdec..c45f350079756282fbb43a1732d256c960f9e274 100644
--- a/src/run_modules/partition_check.py
+++ b/mlair/run_modules/partition_check.py
@@ -1,7 +1,7 @@
__author__ = "Felix Kleinert"
__date__ = '2020-04-07'
-from src.run_modules.run_environment import RunEnvironment
+from mlair.run_modules.run_environment import RunEnvironment
class PartitionCheck(RunEnvironment):
diff --git a/src/run_modules/post_processing.py b/mlair/run_modules/post_processing.py
similarity index 98%
rename from src/run_modules/post_processing.py
rename to mlair/run_modules/post_processing.py
index dedcda0a6a3ff8fb9246bc6efe097eeb6b463999..f096d3b58290d8f6816962d302b5e7a10223c864 100644
--- a/src/run_modules/post_processing.py
+++ b/mlair/run_modules/post_processing.py
@@ -13,14 +13,14 @@ import numpy as np
import pandas as pd
import xarray as xr
-from src.data_handling import BootStraps, Distributor, DataGenerator, DataPrep
-from src.helpers.datastore import NameNotFoundInDataStore
-from src.helpers import TimeTracking, statistics
-from src.model_modules.linear_model import OrdinaryLeastSquaredModel
-from src.model_modules.model_class import AbstractModelClass
-from src.plotting.postprocessing_plotting import PlotMonthlySummary, PlotStationMap, PlotClimatologicalSkillScore, \
+from mlair.data_handling import BootStraps, Distributor, DataGenerator, DataPrep
+from mlair.helpers.datastore import NameNotFoundInDataStore
+from mlair.helpers import TimeTracking, statistics
+from mlair.model_modules.linear_model import OrdinaryLeastSquaredModel
+from mlair.model_modules.model_class import AbstractModelClass
+from mlair.plotting.postprocessing_plotting import PlotMonthlySummary, PlotStationMap, PlotClimatologicalSkillScore, \
PlotCompetitiveSkillScore, PlotTimeSeries, PlotBootstrapSkillScore, PlotAvailability, PlotConditionalQuantiles
-from src.run_modules.run_environment import RunEnvironment
+from mlair.run_modules.run_environment import RunEnvironment
class PostProcessing(RunEnvironment):
diff --git a/src/run_modules/pre_processing.py b/mlair/run_modules/pre_processing.py
similarity index 98%
rename from src/run_modules/pre_processing.py
rename to mlair/run_modules/pre_processing.py
index b4b36a20bf9ed7827a1ac151141c13122f517e33..c5955fc001dd27000e6e50d146eef129ad52f54a 100644
--- a/src/run_modules/pre_processing.py
+++ b/mlair/run_modules/pre_processing.py
@@ -10,11 +10,11 @@ from typing import Tuple, Dict, List
import numpy as np
import pandas as pd
-from src.data_handling import DataGenerator
-from src.helpers import TimeTracking
-from src.configuration import path_config
-from src.helpers.join import EmptyQueryResult
-from src.run_modules.run_environment import RunEnvironment
+from mlair.data_handling import DataGenerator
+from mlair.helpers import TimeTracking
+from mlair.configuration import path_config
+from mlair.helpers.join import EmptyQueryResult
+from mlair.run_modules.run_environment import RunEnvironment
DEFAULT_ARGS_LIST = ["data_path", "network", "stations", "variables", "interpolate_dim", "target_dim", "target_var"]
DEFAULT_KWARGS_LIST = ["limit_nan_fill", "window_history_size", "window_lead_time", "statistics_per_var", "min_length",
diff --git a/src/run_modules/run_environment.py b/mlair/run_modules/run_environment.py
similarity index 86%
rename from src/run_modules/run_environment.py
rename to mlair/run_modules/run_environment.py
index 45d0a4a019b305d477838bd9ec4c5b6f920ac6fb..ecb55282f25c369d6f5eddd81907a7d28ec7d62b 100644
--- a/src/run_modules/run_environment.py
+++ b/mlair/run_modules/run_environment.py
@@ -9,11 +9,11 @@ import os
import shutil
import time
-from src.helpers.datastore import DataStoreByScope as DataStoreObject
-from src.helpers.datastore import NameNotFoundInDataStore
-from src.helpers import Logger
-from src.helpers import TimeTracking
-from src.plotting.tracker_plot import TrackPlot
+from mlair.helpers.datastore import DataStoreByScope as DataStoreObject
+from mlair.helpers.datastore import NameNotFoundInDataStore
+from mlair.helpers import Logger
+from mlair.helpers import TimeTracking
+from mlair.plotting.tracker_plot import TrackPlot
class RunEnvironment(object):
@@ -88,12 +88,16 @@ class RunEnvironment(object):
# set data store and logger (both are mutable!)
del_by_exit = False
- data_store = DataStoreObject()
- logger = Logger()
+ data_store = None
+ logger = None
tracker_list = []
def __init__(self):
"""Start time tracking automatically and logs as info."""
+ if RunEnvironment.data_store is None:
+ RunEnvironment.data_store = DataStoreObject()
+ if RunEnvironment.logger is None:
+ RunEnvironment.logger = Logger()
self.time = TimeTracking()
logging.info(f"{self.__class__.__name__} started")
# atexit.register(self.__del__)
@@ -117,7 +121,7 @@ class RunEnvironment(object):
try:
self.__plot_tracking()
self.__save_tracking()
- self.__copy_log_file()
+ self.__move_log_file()
except FileNotFoundError:
pass
self.data_store.clear_data_store()
@@ -132,11 +136,15 @@ class RunEnvironment(object):
logging.error(exc_val, exc_info=(exc_type, exc_val, exc_tb))
self.__del__()
- def __copy_log_file(self):
+ def __move_log_file(self):
try:
new_file = self.__find_file_pattern("logging_%03i.log")
- logging.info(f"Copy log file to {new_file}")
- shutil.copyfile(self.logger.log_file, new_file)
+ logging.info(f"Move log file to {new_file}")
+ shutil.move(self.logger.log_file, new_file)
+ try:
+ os.rmdir(os.path.dirname(self.logger.log_file))
+ except (OSError, FileNotFoundError):
+ pass
except (NameNotFoundInDataStore, FileNotFoundError):
pass
diff --git a/src/run_modules/training.py b/mlair/run_modules/training.py
similarity index 97%
rename from src/run_modules/training.py
rename to mlair/run_modules/training.py
index 1a0d7beb1ec37bb5e59a4129da58572d79a73636..0c516c577201c62badd3d2b769a291612dce0c5a 100644
--- a/src/run_modules/training.py
+++ b/mlair/run_modules/training.py
@@ -11,10 +11,10 @@ from typing import Union
import keras
from keras.callbacks import Callback, History
-from src.data_handling import Distributor
-from src.model_modules.keras_extensions import CallbackHandler
-from src.plotting.training_monitoring import PlotModelHistory, PlotModelLearningRate
-from src.run_modules.run_environment import RunEnvironment
+from mlair.data_handling import Distributor
+from mlair.model_modules.keras_extensions import CallbackHandler
+from mlair.plotting.training_monitoring import PlotModelHistory, PlotModelLearningRate
+from mlair.run_modules.run_environment import RunEnvironment
class Training(RunEnvironment):
diff --git a/src/run.py b/mlair/run_script.py
similarity index 97%
rename from src/run.py
rename to mlair/run_script.py
index 1494be0a75ebd1fd6a057d337ec2dcef8ed9c64f..2d1f6aeee89c32da56b088a22b06a4b03a58674b 100644
--- a/src/run.py
+++ b/mlair/run_script.py
@@ -1,7 +1,7 @@
__author__ = "Lukas Leufen"
__date__ = '2020-06-29'
-from src.workflows import DefaultWorkflow
+from mlair.workflows import DefaultWorkflow
import inspect
diff --git a/mlair/workflows/__init__.py b/mlair/workflows/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..27c060f10975d86aa35c1f2d45e66966002ecd63
--- /dev/null
+++ b/mlair/workflows/__init__.py
@@ -0,0 +1,2 @@
+from mlair.workflows.abstract_workflow import Workflow
+from mlair.workflows.default_workflow import DefaultWorkflow, DefaultWorkflowHPC
\ No newline at end of file
diff --git a/src/workflows/abstract_workflow.py b/mlair/workflows/abstract_workflow.py
similarity index 97%
rename from src/workflows/abstract_workflow.py
rename to mlair/workflows/abstract_workflow.py
index 5d4e62c8a2e409e865f43412a6757a9cb4e4b1f3..f187ff11e849960b4a63eddd5d11e2ce1ddf2a11 100644
--- a/src/workflows/abstract_workflow.py
+++ b/mlair/workflows/abstract_workflow.py
@@ -5,7 +5,7 @@ __date__ = '2020-06-26'
from collections import OrderedDict
-from src import RunEnvironment
+from mlair import RunEnvironment
class Workflow:
diff --git a/src/workflows/default_workflow.py b/mlair/workflows/default_workflow.py
similarity index 94%
rename from src/workflows/default_workflow.py
rename to mlair/workflows/default_workflow.py
index 6a60c6ae60808b616fded3475179371763fd6feb..c1a9c749865898c3eaa5493ee289602963025464 100644
--- a/src/workflows/default_workflow.py
+++ b/mlair/workflows/default_workflow.py
@@ -4,9 +4,9 @@ __author__ = "Lukas Leufen"
__date__ = '2020-06-26'
import inspect
-from src.helpers import remove_items
-from src.run_modules import ExperimentSetup, PreProcessing, PartitionCheck, ModelSetup, Training, PostProcessing
-from src.workflows.abstract_workflow import Workflow
+from mlair.helpers import remove_items
+from mlair.run_modules import ExperimentSetup, PreProcessing, PartitionCheck, ModelSetup, Training, PostProcessing
+from mlair.workflows.abstract_workflow import Workflow
class DefaultWorkflow(Workflow):
diff --git a/run.py b/run.py
index a9d8190628e1692c4b2812d3c8790bccd6b1b589..15f30a7ee775948fa744832a464562cd40c3e460 100644
--- a/run.py
+++ b/run.py
@@ -2,7 +2,7 @@ __author__ = "Lukas Leufen"
__date__ = '2020-06-29'
import argparse
-from src.workflows import DefaultWorkflow
+from mlair.workflows import DefaultWorkflow
def main(parser_args):
diff --git a/run_HPC.py b/run_HPC.py
index fc2ead406469f0a254f5819e43c1e0d3542bb8d9..d6dbb4dc61e88a1e139b3cbe549bc6a3f2f0ab8a 100644
--- a/run_HPC.py
+++ b/run_HPC.py
@@ -2,7 +2,7 @@ __author__ = "Lukas Leufen"
__date__ = '2020-06-29'
import argparse
-from src.workflows import DefaultWorkflowHPC
+from mlair.workflows import DefaultWorkflowHPC
def main(parser_args):
diff --git a/run_hourly.py b/run_hourly.py
index 682988f6f730d02be713c074dd63fc732e2868dc..b831cf1e1ee733a3c652c6cea364013b44cf2c0d 100644
--- a/run_hourly.py
+++ b/run_hourly.py
@@ -3,7 +3,7 @@ __date__ = '2019-11-14'
import argparse
-from src.workflows import DefaultWorkflow
+from mlair.workflows import DefaultWorkflow
def main(parser_args):
diff --git a/run_zam347.py b/run_zam347.py
index 2d351a8925e67b0bdfc010e92a3937435e160b2f..9027bec807ad9beafcdac573a70aa32d34491034 100644
--- a/run_zam347.py
+++ b/run_zam347.py
@@ -5,13 +5,13 @@ import argparse
import json
import logging
-from src.run_modules.experiment_setup import ExperimentSetup
-from src.run_modules.model_setup import ModelSetup
-from src.run_modules.post_processing import PostProcessing
-from src.run_modules.pre_processing import PreProcessing
-from src.run_modules.run_environment import RunEnvironment
-from src.run_modules.training import Training
-from src.workflows import DefaultWorkflowHPC
+from mlair.run_modules.experiment_setup import ExperimentSetup
+from mlair.run_modules.model_setup import ModelSetup
+from mlair.run_modules.post_processing import PostProcessing
+from mlair.run_modules.pre_processing import PreProcessing
+from mlair.run_modules.run_environment import RunEnvironment
+from mlair.run_modules.training import Training
+from mlair.workflows import DefaultWorkflowHPC
def load_stations():
diff --git a/setup.py b/setup.py
index 8e08e921f5fb728f7b1758e4bb385efc7d71c29b..f708febb5a70c957a91059d840a1f4e140ad35c0 100644
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
import setuptools
-from src import __version__, __author__, __email__
+from mlair import __version__, __author__, __email__
with open("README.md", "r") as fh:
@@ -9,7 +9,7 @@ with open("README.md", "r") as fh:
setuptools.setup(
- name="mlt",
+ name="mlair",
version=__version__,
author=__author__,
author_email=__email__,
@@ -17,8 +17,7 @@ setuptools.setup(
long_description=long_description,
long_description_content_type="text/markdown",
url="https://gitlab.version.fz-juelich.de/toar/machinelearningtools",
- package_dir={'': 'src'},
- packages=setuptools.find_packages(where="src"),
+ packages=setuptools.find_packages(),
classifiers=[
"Programming Language :: Python :: 3",
"License :: OSI Approved :: MIT License", # to be adjusted
diff --git a/src/run_modules/__init__.py b/src/run_modules/__init__.py
deleted file mode 100644
index 0c70ae4205ff38fdc876538c42c44ca0bc8cb9c0..0000000000000000000000000000000000000000
--- a/src/run_modules/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-from src.run_modules.experiment_setup import ExperimentSetup
-from src.run_modules.model_setup import ModelSetup
-from src.run_modules.partition_check import PartitionCheck
-from src.run_modules.post_processing import PostProcessing
-from src.run_modules.pre_processing import PreProcessing
-from src.run_modules.run_environment import RunEnvironment
-from src.run_modules.training import Training
diff --git a/src/workflows/__init__.py b/src/workflows/__init__.py
deleted file mode 100644
index 57e514cd9ced32fbf1dbb290b1008deffcec52d3..0000000000000000000000000000000000000000
--- a/src/workflows/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from src.workflows.abstract_workflow import Workflow
-from src.workflows.default_workflow import DefaultWorkflow, DefaultWorkflowHPC
\ No newline at end of file
diff --git a/test/test_configuration/test_path_config.py b/test/test_configuration/test_path_config.py
index acb43676cb86ca76aded88aa0d46f62dd78d9992..128ddfceeed53920e6424d8d5d8f6addf5451c44 100644
--- a/test/test_configuration/test_path_config.py
+++ b/test/test_configuration/test_path_config.py
@@ -4,9 +4,9 @@ import os
import mock
import pytest
-from src.configuration import prepare_host, set_experiment_name, set_bootstrap_path, check_path_and_create, \
+from mlair.configuration import prepare_host, set_experiment_name, set_bootstrap_path, check_path_and_create, \
set_experiment_path, ROOT_PATH
-from src.helpers import PyTestRegex
+from mlair.helpers import PyTestRegex
class TestPrepareHost:
@@ -25,13 +25,11 @@ class TestPrepareHost:
@mock.patch("socket.gethostname", return_value="NotExistingHostName")
@mock.patch("getpass.getuser", return_value="zombie21")
- def test_error_handling_unknown_host(self, mock_user, mock_host):
- with pytest.raises(OSError) as e:
- prepare_host()
- assert "unknown host 'NotExistingHostName'" in e.value.args[0]
+ def test_prepare_host_unknown(self, mock_user, mock_host):
+ assert prepare_host() == os.path.join(os.path.abspath(os.getcwd()), 'data', 'daily')
@mock.patch("getpass.getuser", return_value="zombie21")
- @mock.patch("src.configuration.path_config.check_path_and_create", side_effect=PermissionError)
+ @mock.patch("mlair.configuration.path_config.check_path_and_create", side_effect=PermissionError)
@mock.patch("os.path.exists", return_value=False)
def test_error_handling(self, mock_path_exists, mock_cpath, mock_user):
# if "runner-6HmDp9Qd-project-2411-concurrent" not in platform.node():
diff --git a/test/test_data_handling/test_bootstraps.py b/test/test_data_handling/test_bootstraps.py
index 3d32a090e62294793bda3f2421de4f52205f427f..c47ec59af4280cfa72e27f1bce06348e5aa2876f 100644
--- a/test/test_data_handling/test_bootstraps.py
+++ b/test/test_data_handling/test_bootstraps.py
@@ -7,8 +7,8 @@ import numpy as np
import pytest
import xarray as xr
-from src.data_handling.bootstraps import BootStraps, CreateShuffledData, BootStrapGenerator
-from src.data_handling.data_generator import DataGenerator
+from mlair.data_handling.bootstraps import BootStraps, CreateShuffledData, BootStrapGenerator
+from mlair.data_handling.data_generator import DataGenerator
@pytest.fixture
@@ -73,7 +73,7 @@ class TestCreateShuffledData:
return CreateShuffledData(orig_generator, 20, data_path)
@pytest.fixture
- @mock.patch("src.data_handling.bootstraps.CreateShuffledData.create_shuffled_data", return_value=None)
+ @mock.patch("mlair.data_handling.bootstraps.CreateShuffledData.create_shuffled_data", return_value=None)
def shuffled_data_no_creation(self, mock_create_shuffle_data, orig_generator, data_path):
return CreateShuffledData(orig_generator, 20, data_path)
@@ -174,7 +174,7 @@ class TestBootStraps:
return BootStraps(orig_generator, data_path, 20)
@pytest.fixture
- @mock.patch("src.data_handling.bootstraps.CreateShuffledData", return_value=None)
+ @mock.patch("mlair.data_handling.bootstraps.CreateShuffledData", return_value=None)
def bootstrap_no_shuffling(self, mock_create_shuffle_data, orig_generator, data_path):
shutil.rmtree(data_path)
return BootStraps(orig_generator, data_path, 20)
@@ -211,7 +211,7 @@ class TestBootStraps:
assert xr.testing.assert_equal(gen.history, expected.sel(variables=var_others)) is None
assert gen.shuffled.variables == "o3"
- @mock.patch("src.data_handling.data_generator.DataGenerator._load_pickle_data", side_effect=FileNotFoundError)
+ @mock.patch("mlair.data_handling.data_generator.DataGenerator._load_pickle_data", side_effect=FileNotFoundError)
def test_get_generator_different_generator(self, mock_load_pickle, data_path, orig_generator):
BootStraps(orig_generator, data_path, 20) # to create
orig_generator.window_history_size = 4
diff --git a/test/test_data_handling/test_data_distributor.py b/test/test_data_handling/test_data_distributor.py
index 9e2242fed67599cdd53ddd29e4fbd4304425b77e..a1fe2f667f33896d2bd2f4c4ad69713020dc7caf 100644
--- a/test/test_data_handling/test_data_distributor.py
+++ b/test/test_data_handling/test_data_distributor.py
@@ -5,8 +5,8 @@ import keras
import numpy as np
import pytest
-from src.data_handling.data_distributor import Distributor
-from src.data_handling.data_generator import DataGenerator
+from mlair.data_handling.data_distributor import Distributor
+from mlair.data_handling.data_generator import DataGenerator
from test.test_modules.test_training import my_test_model
diff --git a/test/test_data_handling/test_data_generator.py b/test/test_data_handling/test_data_generator.py
index 754728ba403fbda25c021c2f576a1bc89d26f83f..579b34f6b24d126bc10d28a255ed2f6f63662bdd 100644
--- a/test/test_data_handling/test_data_generator.py
+++ b/test/test_data_handling/test_data_generator.py
@@ -6,9 +6,9 @@ import numpy as np
import pytest
import xarray as xr
-from src.data_handling.data_generator import DataGenerator
-from src.data_handling.data_preparation import DataPrep
-from src.helpers.join import EmptyQueryResult
+from mlair.data_handling.data_generator import DataGenerator
+from mlair.data_handling.data_preparation import DataPrep
+from mlair.helpers.join import EmptyQueryResult
class TestDataGenerator:
diff --git a/test/test_data_handling/test_data_preparation.py b/test/test_data_handling/test_data_preparation.py
index a8ca555c9748f7656fefc007922ee0d7df1992fa..4106e4a75a0ecd295a5ec4d2ffcab6c98b7a3b04 100644
--- a/test/test_data_handling/test_data_preparation.py
+++ b/test/test_data_handling/test_data_preparation.py
@@ -8,8 +8,8 @@ import pandas as pd
import pytest
import xarray as xr
-from src.data_handling.data_preparation import DataPrep
-from src.helpers.join import EmptyQueryResult
+from mlair.data_handling.data_preparation import DataPrep
+from mlair.helpers.join import EmptyQueryResult
class TestDataPrep:
diff --git a/test/test_datastore.py b/test/test_datastore.py
index 9aca1eef35927242df0b5f659eece716f81f6c13..662c90bf04e11b8b4ff9647506c1981c8883f30b 100644
--- a/test/test_datastore.py
+++ b/test/test_datastore.py
@@ -3,8 +3,8 @@ __date__ = '2019-11-22'
import pytest
-from src.helpers.datastore import AbstractDataStore, DataStoreByVariable, DataStoreByScope, CorrectScope
-from src.helpers.datastore import NameNotFoundInDataStore, NameNotFoundInScope, EmptyScope
+from mlair.helpers.datastore import AbstractDataStore, DataStoreByVariable, DataStoreByScope, CorrectScope
+from mlair.helpers.datastore import NameNotFoundInDataStore, NameNotFoundInScope, EmptyScope
class TestAbstractDataStore:
diff --git a/test/test_helpers/test_helpers.py b/test/test_helpers/test_helpers.py
index 28a8bf6e421d62d58d76e7a32906f8a594f16ed7..281d60e07463c6b5118f36714d80144443a03050 100644
--- a/test/test_helpers/test_helpers.py
+++ b/test/test_helpers/test_helpers.py
@@ -5,13 +5,14 @@ import datetime as dt
import logging
import math
import time
+import os
import mock
import pytest
-from src.helpers import to_list, dict_to_xarray, float_round, remove_items
-from src.helpers import PyTestRegex
-from src.helpers import Logger, TimeTracking
+from mlair.helpers import to_list, dict_to_xarray, float_round, remove_items
+from mlair.helpers import PyTestRegex
+from mlair.helpers import Logger, TimeTracking
class TestToList:
@@ -236,8 +237,8 @@ class TestLogger:
def test_setup_logging_path_none(self):
log_file = Logger.setup_logging_path(None)
- assert PyTestRegex(
- ".*machinelearningtools/logging/logging_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.log") == log_file
+ test_regex = os.getcwd() + r"/logging/logging_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}\.log"
+ assert PyTestRegex(test_regex) == log_file
@mock.patch("os.makedirs", side_effect=None)
def test_setup_logging_path_given(self, mock_makedirs):
diff --git a/test/test_join.py b/test/test_join.py
index 5adc013cfbd446c4feaf4a2b344f07d6f170077d..791723335e16cf2124512629414ebe626bc20e9c 100644
--- a/test/test_join.py
+++ b/test/test_join.py
@@ -2,9 +2,9 @@ from typing import Iterable
import pytest
-from src.helpers.join import *
-from src.helpers.join import _save_to_pandas, _correct_stat_name, _lower_list
-from src.configuration.join_settings import join_settings
+from mlair.helpers.join import *
+from mlair.helpers.join import _save_to_pandas, _correct_stat_name, _lower_list
+from mlair.configuration.join_settings import join_settings
class TestJoinUrlBase:
diff --git a/test/test_model_modules/test_advanced_paddings.py b/test/test_model_modules/test_advanced_paddings.py
index 8c7cae91ad12cc2b06ec82ba64f91c792a620756..8ca81c42c0b807b28c444badba8d92a255341eb4 100644
--- a/test/test_model_modules/test_advanced_paddings.py
+++ b/test/test_model_modules/test_advanced_paddings.py
@@ -1,7 +1,7 @@
import keras
import pytest
-from src.model_modules.advanced_paddings import *
+from mlair.model_modules.advanced_paddings import *
class TestPadUtils:
diff --git a/test/test_model_modules/test_flatten_tail.py b/test/test_model_modules/test_flatten_tail.py
index 0de138ec2323aea3409d5deadfb26c9741b89f50..623d51c07f6b27c8d6238d8a5189dea33837115e 100644
--- a/test/test_model_modules/test_flatten_tail.py
+++ b/test/test_model_modules/test_flatten_tail.py
@@ -1,6 +1,6 @@
import keras
import pytest
-from src.model_modules.flatten import flatten_tail, get_activation
+from mlair.model_modules.flatten import flatten_tail, get_activation
class TestGetActivation:
diff --git a/test/test_model_modules/test_inception_model.py b/test/test_model_modules/test_inception_model.py
index ca0126a44fa0f8ccd2ed2a7ea79c872c4731fea1..2dfc2c9c1c0510355216769b2ab83152a0a02118 100644
--- a/test/test_model_modules/test_inception_model.py
+++ b/test/test_model_modules/test_inception_model.py
@@ -1,9 +1,9 @@
import keras
import pytest
-from src.helpers import PyTestRegex
-from src.model_modules.advanced_paddings import ReflectionPadding2D, SymmetricPadding2D
-from src.model_modules.inception_model import InceptionModelBase
+from mlair.helpers import PyTestRegex
+from mlair.model_modules.advanced_paddings import ReflectionPadding2D, SymmetricPadding2D
+from mlair.model_modules.inception_model import InceptionModelBase
class TestInceptionModelBase:
diff --git a/test/test_model_modules/test_keras_extensions.py b/test/test_model_modules/test_keras_extensions.py
index 56c60ec43173e9fdd438214862603caba632bc65..78559ee0e54c725d242194133549d8b17699b729 100644
--- a/test/test_model_modules/test_keras_extensions.py
+++ b/test/test_model_modules/test_keras_extensions.py
@@ -4,8 +4,8 @@ import keras
import mock
import pytest
-from src.model_modules.loss import l_p_loss
-from src.model_modules.keras_extensions import *
+from mlair.model_modules.loss import l_p_loss
+from mlair.model_modules.keras_extensions import *
class TestHistoryAdvanced:
diff --git a/test/test_model_modules/test_loss.py b/test/test_model_modules/test_loss.py
index c47f3f188a4b360bda08470fb00fd1d88a9f754c..e54e0b00de4a71d241f30e0b6b0c1a2e8fa1a19c 100644
--- a/test/test_model_modules/test_loss.py
+++ b/test/test_model_modules/test_loss.py
@@ -1,7 +1,7 @@
import keras
import numpy as np
-from src.model_modules.loss import l_p_loss
+from mlair.model_modules.loss import l_p_loss
class TestLoss:
diff --git a/test/test_model_modules/test_model_class.py b/test/test_model_modules/test_model_class.py
index 0ee2eb7e5d439c76888f1f05e238bb5507db6a7a..6025516ba01abdcb35ea65b9c4570d5a8b0928b5 100644
--- a/test/test_model_modules/test_model_class.py
+++ b/test/test_model_modules/test_model_class.py
@@ -1,8 +1,8 @@
import keras
import pytest
-from src.model_modules.model_class import AbstractModelClass
-from src.model_modules.model_class import MyPaperModel
+from mlair.model_modules.model_class import AbstractModelClass
+from mlair.model_modules.model_class import MyPaperModel
class Paddings:
diff --git a/test/test_modules/test_experiment_setup.py b/test/test_modules/test_experiment_setup.py
index 5b7d517e658de6bd71e1b4190bb5114dc005216e..60f0d2f9143b83a1333dd5de347591058686ea5c 100644
--- a/test/test_modules/test_experiment_setup.py
+++ b/test/test_modules/test_experiment_setup.py
@@ -4,9 +4,9 @@ import os
import pytest
-from src.helpers import TimeTracking
-from src.configuration.path_config import prepare_host
-from src.run_modules.experiment_setup import ExperimentSetup
+from mlair.helpers import TimeTracking
+from mlair.configuration.path_config import prepare_host
+from mlair.run_modules.experiment_setup import ExperimentSetup
class TestExperimentSetup:
@@ -14,7 +14,7 @@ class TestExperimentSetup:
@pytest.fixture
def empty_obj(self, caplog):
obj = object.__new__(ExperimentSetup)
- obj.time = TimeTracking()
+ super(ExperimentSetup, obj).__init__()
caplog.set_level(logging.DEBUG)
return obj
@@ -43,7 +43,7 @@ class TestExperimentSetup:
assert data_store.get("fraction_of_training", "general") == 0.8
# set experiment name
assert data_store.get("experiment_name", "general") == "TestExperiment_daily"
- path = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "TestExperiment_daily"))
+ path = os.path.abspath(os.path.join(os.getcwd(), "TestExperiment_daily"))
assert data_store.get("experiment_path", "general") == path
default_statistics_per_var = {'o3': 'dma8eu', 'relhum': 'average_values', 'temp': 'maximum',
'u': 'average_values', 'v': 'average_values', 'no': 'dma8eu', 'no2': 'dma8eu',
diff --git a/test/test_modules/test_model_setup.py b/test/test_modules/test_model_setup.py
index 60d140f8845b25432184de1f3890b3ee4d0b034e..b61e52d21758213c34b7830589f58e1edb53dc77 100644
--- a/test/test_modules/test_model_setup.py
+++ b/test/test_modules/test_model_setup.py
@@ -2,12 +2,12 @@ import os
import pytest
-from src.data_handling.data_generator import DataGenerator
-from src.helpers.datastore import EmptyScope
-from src.model_modules.keras_extensions import CallbackHandler
-from src.model_modules.model_class import AbstractModelClass, MyLittleModel
-from src.run_modules.model_setup import ModelSetup
-from src.run_modules.run_environment import RunEnvironment
+from mlair.data_handling.data_generator import DataGenerator
+from mlair.helpers.datastore import EmptyScope
+from mlair.model_modules.keras_extensions import CallbackHandler
+from mlair.model_modules.model_class import AbstractModelClass, MyLittleModel
+from mlair.run_modules.model_setup import ModelSetup
+from mlair.run_modules.run_environment import RunEnvironment
class TestModelSetup:
diff --git a/test/test_modules/test_partition_check.py b/test/test_modules/test_partition_check.py
index b04e01d13e9e160553f8ff66af8d97f65aa24bf0..1e576a8ce47c98e395468b76d3496dafa3cc0525 100644
--- a/test/test_modules/test_partition_check.py
+++ b/test/test_modules/test_partition_check.py
@@ -2,10 +2,10 @@ import logging
import pytest
import mock
-from src.run_modules.experiment_setup import ExperimentSetup
-from src.run_modules.partition_check import PartitionCheck
-from src.run_modules.run_environment import RunEnvironment
-from src.configuration import get_host
+from mlair.run_modules.experiment_setup import ExperimentSetup
+from mlair.run_modules.partition_check import PartitionCheck
+from mlair.run_modules.run_environment import RunEnvironment
+from mlair.configuration import get_host
class TestPartitionCheck:
diff --git a/test/test_modules/test_pre_processing.py b/test/test_modules/test_pre_processing.py
index 29811fb04789f32a1e2cc1b3affb6f8d4ae99730..93f322f10820348ea3b431baccfc7404079c8fe8 100644
--- a/test/test_modules/test_pre_processing.py
+++ b/test/test_modules/test_pre_processing.py
@@ -2,12 +2,12 @@ import logging
import pytest
-from src.data_handling.data_generator import DataGenerator
-from src.helpers.datastore import NameNotFoundInScope
-from src.helpers import PyTestRegex
-from src.run_modules.experiment_setup import ExperimentSetup
-from src.run_modules.pre_processing import PreProcessing, DEFAULT_ARGS_LIST, DEFAULT_KWARGS_LIST
-from src.run_modules.run_environment import RunEnvironment
+from mlair.data_handling.data_generator import DataGenerator
+from mlair.helpers.datastore import NameNotFoundInScope
+from mlair.helpers import PyTestRegex
+from mlair.run_modules.experiment_setup import ExperimentSetup
+from mlair.run_modules.pre_processing import PreProcessing, DEFAULT_ARGS_LIST, DEFAULT_KWARGS_LIST
+from mlair.run_modules.run_environment import RunEnvironment
class TestPreProcessing:
diff --git a/test/test_modules/test_run_environment.py b/test/test_modules/test_run_environment.py
index 59bb8535c4dab44e646bd6bc4aa83a8553be4d26..aa385e32673c2bf58db3f5666b2f64076af0193f 100644
--- a/test/test_modules/test_run_environment.py
+++ b/test/test_modules/test_run_environment.py
@@ -1,7 +1,7 @@
import logging
-from src.helpers import TimeTracking, PyTestRegex
-from src.run_modules.run_environment import RunEnvironment
+from mlair.helpers import TimeTracking, PyTestRegex
+from mlair.run_modules.run_environment import RunEnvironment
class TestRunEnvironment:
diff --git a/test/test_modules/test_training.py b/test/test_modules/test_training.py
index eb5dfe5adb170981d5d67c94ca1fbcb55e326550..292bf7e76deb587b7a64e31e93a67cb143d8e540 100644
--- a/test/test_modules/test_training.py
+++ b/test/test_modules/test_training.py
@@ -9,14 +9,14 @@ import mock
import pytest
from keras.callbacks import History
-from src.data_handling.data_distributor import Distributor
-from src.data_handling.data_generator import DataGenerator
-from src.helpers import PyTestRegex
-from src.model_modules.flatten import flatten_tail
-from src.model_modules.inception_model import InceptionModelBase
-from src.model_modules.keras_extensions import LearningRateDecay, HistoryAdvanced, CallbackHandler
-from src.run_modules.run_environment import RunEnvironment
-from src.run_modules.training import Training
+from mlair.data_handling.data_distributor import Distributor
+from mlair.data_handling.data_generator import DataGenerator
+from mlair.helpers import PyTestRegex
+from mlair.model_modules.flatten import flatten_tail
+from mlair.model_modules.inception_model import InceptionModelBase
+from mlair.model_modules.keras_extensions import LearningRateDecay, HistoryAdvanced, CallbackHandler
+from mlair.run_modules.run_environment import RunEnvironment
+from mlair.run_modules.training import Training
def my_test_model(activation, window_history_size, channels, dropout_rate, add_minor_branch=False):
diff --git a/test/test_plotting/test_tracker_plot.py b/test/test_plotting/test_tracker_plot.py
index 9a92360a819c130c213d06b89a48a896e082adad..196879657452fe12238c990fc419cb0848c9ec9c 100644
--- a/test/test_plotting/test_tracker_plot.py
+++ b/test/test_plotting/test_tracker_plot.py
@@ -7,8 +7,8 @@ import shutil
from matplotlib import pyplot as plt
import numpy as np
-from src.plotting.tracker_plot import TrackObject, TrackChain, TrackPlot
-from src.helpers import PyTestAllEqual
+from mlair.plotting.tracker_plot import TrackObject, TrackChain, TrackPlot
+from mlair.helpers import PyTestAllEqual
class TestTrackObject:
diff --git a/test/test_plotting/test_training_monitoring.py b/test/test_plotting/test_training_monitoring.py
index 6e5e0abbc5da0978e200f19019700c4dedd14ad0..18009bc19947bd3318c6f1d220d303c1efeec972 100644
--- a/test/test_plotting/test_training_monitoring.py
+++ b/test/test_plotting/test_training_monitoring.py
@@ -3,8 +3,8 @@ import os
import keras
import pytest
-from src.model_modules.keras_extensions import LearningRateDecay
-from src.plotting.training_monitoring import PlotModelLearningRate, PlotModelHistory
+from mlair.model_modules.keras_extensions import LearningRateDecay
+from mlair.plotting.training_monitoring import PlotModelLearningRate, PlotModelHistory
@pytest.fixture
diff --git a/test/test_statistics.py b/test/test_statistics.py
index 3da7a47871f6d92472de268d165d788c343ce394..d4a72674ae89ecd106ff1861aa6ee26567da3243 100644
--- a/test/test_statistics.py
+++ b/test/test_statistics.py
@@ -3,7 +3,7 @@ import pandas as pd
import pytest
import xarray as xr
-from src.helpers.statistics import standardise, standardise_inverse, standardise_apply, centre, centre_inverse, centre_apply, \
+from mlair.helpers.statistics import standardise, standardise_inverse, standardise_apply, centre, centre_inverse, centre_apply, \
apply_inverse_transformation
lazy = pytest.lazy_fixture