diff --git a/README.md b/README.md
index 5c55b4094232908a56cdcf61ba437976f8714e8b..c33aab4b8643d2907b07b5ebcb254076515d03d2 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,10 @@
# MLAir - Machine Learning on Air Data
MLAir (Machine Learning on Air data) is an environment that simplifies and accelerates the creation of new machine
-learning (ML) models for the analysis and forecasting of meteorological and air quality time series.
+learning (ML) models for the analysis and forecasting of meteorological and air quality time series. You can find the
+docs [here](http://toar.pages.jsc.fz-juelich.de/mlair/docs/).
+
+[[_TOC_]]
# Installation
@@ -9,7 +12,8 @@ MLAir is based on several python frameworks. To work properly, you have to insta
`requirements.txt` file. Additionally to support the geographical plotting part it is required to install geo
packages built for your operating system. Name names of these package may differ for different systems, we refer
here to the opensuse / leap OS. The geo plot can be removed from the `plot_list`, in this case there is no need to
-install the geo packages.
+install the geo packages. For special instructions to install MLAir on the Juelich HPC systems, see
+[here](#special-instructions-for-installation-on-jülich-hpc-systems).
* (geo) Install **proj** on your machine using the console. E.g. for opensuse / leap `zypper install proj`
* (geo) A c++ compiler is required for the installation of the program **cartopy**
@@ -27,7 +31,9 @@ install the geo packages.
# How to start with MLAir
-In this section, we show three examples how to work with MLAir.
+In this section, we show three examples how to work with MLAir. Note, that for these examples MLAir was installed using
+the distribution file. In case you are using the git clone it is required to adjust the import path if not directly
+executed inside the source directory of MLAir.
## Example 1
@@ -112,12 +118,50 @@ INFO: No training has started, because trainable parameter was false.
INFO: mlair finished after 00:00:06 (hh:mm:ss)
```
-# Customised workflows and models
-# Custom Workflow
+# Default Workflow
+
+MLAir is constituted of so-called `run_modules` which are executed in a distinct order called `workflow`. MLAir
+provides a `default_workflow`. This workflow runs the run modules `ExperimentSetup`, `PreProcessing`,
+`ModelSetup`, `Training`, and `PostProcessing` one by one.
-MLAir provides a default workflow. If additional steps are to be performed, you have to append custom run modules to
-the workflow.
+
+
+```python
+import mlair
+
+# create your custom MLAir workflow
+DefaultWorkflow = mlair.DefaultWorkflow()
+# execute default workflow
+DefaultWorkflow.run()
+```
+The output of running this default workflow will be structured like the following.
+```log
+INFO: mlair started
+INFO: ExperimentSetup started
+...
+INFO: ExperimentSetup finished after 00:00:01 (hh:mm:ss)
+INFO: PreProcessing started
+...
+INFO: PreProcessing finished after 00:00:11 (hh:mm:ss)
+INFO: ModelSetup started
+...
+INFO: ModelSetup finished after 00:00:01 (hh:mm:ss)
+INFO: Training started
+...
+INFO: Training finished after 00:02:15 (hh:mm:ss)
+INFO: PostProcessing started
+...
+INFO: PostProcessing finished after 00:01:37 (hh:mm:ss)
+INFO: mlair finished after 00:04:05 (hh:mm:ss)
+```
+
+# Customised Run Module and Workflow
+
+It is possible to create new custom run modules. A custom run module is required to inherit from the base class
+`RunEnvironment` and to hold the constructor method `__init__()`. This method has to execute the module on call.
+In the following example, this is done by using the `_run()` method that is called by the initialiser. It is
+possible to parse arguments to the custom run module as shown.
```python
import mlair
@@ -129,14 +173,19 @@ class CustomStage(mlair.RunEnvironment):
def __init__(self, test_string):
super().__init__() # always call super init method
self._run(test_string) # call a class method
-
+
def _run(self, test_string):
logging.info("Just running a custom stage.")
logging.info("test_string = " + test_string)
epochs = self.data_store.get("epochs")
logging.info("epochs = " + str(epochs))
+```
+
+If a custom run module is defined, it is required to adjust the workflow. For this, you need to load the empty
+`Workflow` class and add each run module that is required. The order of adding modules defines the order of
+execution if running the workflow.
-
+```python
# create your custom MLAir workflow
CustomWorkflow = mlair.Workflow()
# provide stages without initialisation
@@ -146,6 +195,9 @@ CustomWorkflow.add(CustomStage, test_string="Hello World")
# finally execute custom workflow in order of adding
CustomWorkflow.run()
```
+
+The output will look like:
+
```log
INFO: mlair started
...
@@ -158,115 +210,198 @@ INFO: CustomStage finished after 00:00:01 (hh:mm:ss)
INFO: mlair finished after 00:00:13 (hh:mm:ss)
```
-## Custom Model
+# Custom Model
-Each model has to inherit from the abstract model class to ensure a smooth training and evaluation behaviour. It is
-required to implement the set model and set compile options methods. The later has to set the loss at least.
+Create your own model to run your personal experiment. To guarantee a proper integration in the MLAir workflow, models
+are restricted to inherit from the `AbstractModelClass`. This will ensure a smooth training and evaluation
+behaviour.
-```python
+## How to create a customised model?
+* Create a new model class inheriting from `AbstractModelClass`
+
+```python
+from mlair import AbstractModelClass
import keras
-from keras.losses import mean_squared_error as mse
-from keras.optimizers import SGD
-from mlair.model_modules import AbstractModelClass
+class MyCustomisedModel(AbstractModelClass):
+
+ def __init__(self, shape_inputs: list, shape_outputs: list):
+
+ super().__init__(shape_inputs[0], shape_outputs[0])
-class MyLittleModel(AbstractModelClass):
- """
- A customised model with a 1x1 Conv, and 3 Dense layers (32, 16
- window_lead_time). Dropout is used after Conv layer.
- """
- def __init__(self, window_history_size, window_lead_time, channels):
- super().__init__()
# settings
- self.window_history_size = window_history_size
- self.window_lead_time = window_lead_time
- self.channels = channels
self.dropout_rate = 0.1
self.activation = keras.layers.PReLU
- self.lr = 1e-2
+
# apply to model
self.set_model()
self.set_compile_options()
self.set_custom_objects(loss=self.compile_options['loss'])
+```
+
+* Make sure to add the `super().__init__()` and at least `set_model()` and `set_compile_options()` to your
+ custom init method.
+* The shown model expects a single input and output branch provided in a list. Therefore shapes of input and output are
+ extracted and then provided to the super class initialiser.
+* Some general settings like the dropout rate are set in the init method additionally.
+* If you have custom objects in your model, that are not part of the keras or tensorflow frameworks, you need to add
+ them to custom objects. To do this, call `set_custom_objects` with arbitrarily kwargs. In the shown example, the
+ loss has been added for demonstration only, because we use a build-in loss function. Nonetheless, we always encourage
+ you to add the loss as custom object, to prevent potential errors when loading an already created model instead of
+ training a new one.
+* Now build your model inside `set_model()` by using the instance attributes `self.shape_inputs` and
+ `self.shape_outputs` and storing the model as `self.model`.
+
+```python
+class MyCustomisedModel(AbstractModelClass):
def set_model(self):
- # add 1 to window_size to include current time step t0
- shape = (self.window_history_size + 1, 1, self.channels)
- x_input = keras.layers.Input(shape=shape)
- x_in = keras.layers.Conv2D(32, (1, 1), padding='same')(x_input)
+ x_input = keras.layers.Input(shape=self.shape_inputs)
+ x_in = keras.layers.Conv2D(32, (1, 1), padding='same', name='{}_Conv_1x1'.format("major"))(x_input)
+ x_in = self.activation(name='{}_conv_act'.format("major"))(x_in)
+ x_in = keras.layers.Flatten(name='{}'.format("major"))(x_in)
+ x_in = keras.layers.Dropout(self.dropout_rate, name='{}_Dropout_1'.format("major"))(x_in)
+ x_in = keras.layers.Dense(16, name='{}_Dense_16'.format("major"))(x_in)
x_in = self.activation()(x_in)
- x_in = keras.layers.Flatten()(x_in)
- x_in = keras.layers.Dropout(self.dropout_rate)(x_in)
- x_in = keras.layers.Dense(32)(x_in)
- x_in = self.activation()(x_in)
- x_in = keras.layers.Dense(16)(x_in)
- x_in = self.activation()(x_in)
- x_in = keras.layers.Dense(self.window_lead_time)(x_in)
- out = self.activation()(x_in)
- self.model = keras.Model(inputs=x_input, outputs=[out])
+ x_in = keras.layers.Dense(self.shape_outputs, name='{}_Dense'.format("major"))(x_in)
+ out_main = self.activation()(x_in)
+ self.model = keras.Model(inputs=x_input, outputs=[out_main])
+```
+
+* Your are free how to design your model. Just make sure to save it in the class attribute model.
+* Additionally, set your custom compile options including the loss definition.
+
+```python
+class MyCustomisedModel(AbstractModelClass):
def set_compile_options(self):
- self.compile_options = {"optimizer": SGD(lr=self.lr),
- "loss": mse,
- "metrics": ["mse"]}
+ self.initial_lr = 1e-2
+ self.optimizer = keras.optimizers.SGD(lr=self.initial_lr, momentum=0.9)
+ self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr,
+ drop=.94,
+ epochs_drop=10)
+ self.loss = keras.losses.mean_squared_error
+ self.compile_options = {"metrics": ["mse", "mae"]}
```
+* The allocation of the instance parameters `initial_lr`, `optimizer`, and `lr_decay` could be also part of
+ the model class' initialiser. The same applies to `self.loss` and `compile_options`, but we recommend to use
+ the `set_compile_options` method for the definition of parameters, that are related to the compile options.
+* More important is that the compile options are actually saved. There are three ways to achieve this.
+
+ * (1): Set all compile options by parsing a dictionary with all options to `self.compile_options`.
+ * (2): Set all compile options as instance attributes. MLAir will search for these attributes and store them.
+ * (3): Define your compile options partly as dictionary and instance attributes (as shown in this example).
+ * If using (3) and defining the same compile option with different values, MLAir will raise an error.
+
+ Incorrect: (Will raise an error because of a mismatch for the `optimizer` parameter.)
+ ```python
+ def set_compile_options(self):
+ self.optimizer = keras.optimizers.SGD()
+ self.loss = keras.losses.mean_squared_error
+ self.compile_options = {"optimizer" = keras.optimizers.Adam()}
+ ```
+
+
+## Specials for Branched Models
+
+* If you have a branched model with multiple outputs, you need either set only a single loss for all branch outputs or
+ provide the same number of loss functions considering the right order.
-## Transformation
+```python
+class MyCustomisedModel(AbstractModelClass):
-There are two different approaches (called scopes) to transform the data:
-1) `station`: transform data for each station independently (somehow like batch normalisation)
-1) `data`: transform all data of each station with shared metrics
+ def set_model(self):
+ ...
+ self.model = keras.Model(inputs=x_input, outputs=[out_minor_1, out_minor_2, out_main])
-Transformation must be set by the `transformation` attribute. If `transformation = None` is given to `ExperimentSetup`,
-data is not transformed at all. For all other setups, use the following dictionary structure to specify the
-transformation.
+ def set_compile_options(self):
+ self.loss = [keras.losses.mean_absolute_error] + # for out_minor_1
+ [keras.losses.mean_squared_error] + # for out_minor_2
+ [keras.losses.mean_squared_error] # for out_main
```
-transformation = {"scope": <...>,
- "method": <...>,
- "mean": <...>,
- "std": <...>}
-ExperimentSetup(..., transformation=transformation, ...)
+
+
+## How to access my customised model?
+
+If the customised model is created, you can easily access the model with
+
+```python
+>>> MyCustomisedModel().model
+<your custom model>
```
-### scopes
+The loss is accessible via
-**station**: mean and std are not used
+```python
+>>> MyCustomisedModel().loss
+<your custom loss>
+```
-**data**: either provide already calculated values for mean and std (if required by transformation method), or choose
-from different calculation schemes, explained in the mean and std section.
+You can treat the instance of your model as instance but also as the model itself. If you call a method, that refers to
+the model instead of the model instance, you can directly apply the command on the instance instead of adding the model
+parameter call.
-### supported transformation methods
-Currently supported methods are:
-* standardise (default, if method is not given)
-* centre
+```python
+>>> MyCustomisedModel().model.compile(**kwargs) == MyCustomisedModel().compile(**kwargs)
+True
+```
-### mean and std
-`"mean"="accurate"`: calculate the accurate values of mean and std (depending on method) by using all data. Although,
-this method is accurate, it may take some time for the calculation. Furthermore, this could potentially lead to memory
-issue (not explored yet, but could appear for a very big amount of data)
+# Data Handlers
-`"mean"="estimate"`: estimate mean and std (depending on method). For each station, mean and std are calculated and
-afterwards aggregated using the mean value over all station-wise metrics. This method is less accurate, especially
-regarding the std calculation but therefore much faster.
+Data handlers are responsible for all tasks related to data like data acquisition, preparation and provision. A data
+handler must inherit from the abstract base class `AbstractDataHandler` and requires the implementation of the
+`__init__()` method and the accessors `get_X()` and `get_Y()`. In the following, we show an example how a custom data
+handler could look like.
-We recommend to use the later method *estimate* because of following reasons:
-* much faster calculation
-* real accuracy of mean and std is less important, because it is "just" a transformation / scaling
-* accuracy of mean is almost as high as in the *accurate* case, because of
-$\bar{x_{ij}} = \bar{\left(\bar{x_i}\right)_j}$. The only difference is, that in the *estimate* case, each mean is
-equally weighted for each station independently of the actual data count of the station.
-* accuracy of std is lower for *estimate* because of $\var{x_{ij}} \ne \bar{\left(\var{x_i}\right)_j}$, but still the mean of all
-station-wise std is a decent estimate of the true std.
+```python
+import datetime as dt
+import numpy as np
+import pandas as pd
+import xarray as xr
-`"mean"=<value, e.g. xr.DataArray>`: If mean and std are already calculated or shall be set manually, just add the
-scaling values instead of the calculation method. For method *centre*, std can still be None, but is required for the
-*standardise* method. **Important**: Format of given values **must** match internal data format of DataPreparation
-class: `xr.DataArray` with `dims=["variables"]` and one value for each variable.
+from mlair.data_handler import AbstractDataHandler
+class DummyDataHandler(AbstractDataHandler):
+ def __init__(self, name, number_of_samples=None):
+ """This data handler takes a name argument and the number of samples to generate. If not provided, a random
+ number between 100 and 150 is set."""
+ super().__init__()
+ self.name = name
+ self.number_of_samples = number_of_samples if number_of_samples is not None else np.random.randint(100, 150)
+ self._X = self.create_X()
+ self._Y = self.create_Y()
+
+ def create_X(self):
+ """Inputs are random numbers between 0 and 10 with shape (no_samples, window=14, variables=5)."""
+ X = np.random.randint(0, 10, size=(self.number_of_samples, 14, 5)) # samples, window, variables
+ datelist = pd.date_range(dt.datetime.today().date(), periods=self.number_of_samples, freq="H").tolist()
+ return xr.DataArray(X, dims=['datetime', 'window', 'variables'], coords={"datetime": datelist,
+ "window": range(14),
+ "variables": range(5)})
+
+ def create_Y(self):
+ """Targets are normal distributed random numbers with shape (no_samples, window=5, variables=1)."""
+ Y = np.round(0.5 * np.random.randn(self.number_of_samples, 5, 1), 1) # samples, window, variables
+ datelist = pd.date_range(dt.datetime.today().date(), periods=self.number_of_samples, freq="H").tolist()
+ return xr.DataArray(Y, dims=['datetime', 'window', 'variables'], coords={"datetime": datelist,
+ "window": range(5),
+ "variables": range(1)})
+
+ def get_X(self, upsampling=False, as_numpy=False):
+ """Upsampling parameter is not used for X."""
+ return np.copy(self._X) if as_numpy is True else self._X
+
+ def get_Y(self, upsampling=False, as_numpy=False):
+ """Upsampling parameter is not used for Y."""
+ return np.copy(self._Y) if as_numpy is True else self._Y
+
+ def __str__(self):
+ return self.name
+```
# Special Remarks
@@ -297,3 +432,4 @@ Therefore, it might be necessary to adopt the `if` statement in `PartitionCheck.
add it to `src/join_settings.py` in the hourly data section. Replace the `TOAR_SERVICE_URL` and the `Authorization`
value. To make sure, that this **sensitive** data is not uploaded to the remote server, use the following command to
prevent git from tracking this file: `git update-index --assume-unchanged src/join_settings.py`
+
diff --git a/docs/_source/_api/machinelearningtools.rst b/docs/_source/_api/mlair.rst
similarity index 50%
rename from docs/_source/_api/machinelearningtools.rst
rename to docs/_source/_api/mlair.rst
index cd6885f52bedfa295139251c641c5bba8e2a30e9..26166fc3aa8d824e2b0a80a19c656596a86ca3c0 100644
--- a/docs/_source/_api/machinelearningtools.rst
+++ b/docs/_source/_api/mlair.rst
@@ -1,7 +1,7 @@
-machinelearningtools package
-============================
+MLAir package
+=============
-.. automodule:: src
+.. automodule:: mlair
:members:
:undoc-members:
:show-inheritance:
diff --git a/docs/_source/_plots/run_modules_schedule.png b/docs/_source/_plots/run_modules_schedule.png
new file mode 100755
index 0000000000000000000000000000000000000000..b7549849757118a688bad0c4128b6de694b079bc
Binary files /dev/null and b/docs/_source/_plots/run_modules_schedule.png differ
diff --git a/docs/_source/conf.py b/docs/_source/conf.py
index 573918ee35e9757b8c0b32b2697fc0cc2bc0b38f..d4c71e69cc38499930d28952562372106b5f13b3 100644
--- a/docs/_source/conf.py
+++ b/docs/_source/conf.py
@@ -15,6 +15,8 @@ import sys
sys.path.insert(0, os.path.abspath('../..'))
+import mlair
+
# -- Project information -----------------------------------------------------
project = 'MLAir'
@@ -22,9 +24,9 @@ copyright = '2020, Lukas H Leufen, Felix Kleinert'
author = 'Lukas H Leufen, Felix Kleinert'
# The short X.Y version
-version = 'v0.9.0'
+version = "v" + ".".join(mlair.__version__.split(".")[0:2])
# The full version, including alpha/beta/rc tags
-release = 'v0.9.0'
+release = "v" + mlair.__version__
# -- General configuration ---------------------------------------------------
diff --git a/docs/_source/customise.rst b/docs/_source/customise.rst
new file mode 100644
index 0000000000000000000000000000000000000000..4c9ee5386365c74e3d85c0f81085fcd3e1971b69
--- /dev/null
+++ b/docs/_source/customise.rst
@@ -0,0 +1,353 @@
+Default Workflow
+----------------
+
+.. role:: py(code)
+ :language: python
+
+MLAir is constituted of so-called :py:`run_modules` which are executed in a distinct order called :py:`workflow`. MLAir
+provides a :py:`DefaultWorkflow`. This workflow runs the run modules :py:`ExperimentSetup`, :py:`PreProcessing`,
+:py:`ModelSetup`, :py:`Training`, and :py:`PostProcessing` one by one.
+
+
+.. figure:: ./_plots/run_modules_schedule.png
+
+ Sketch of the default workflow.
+
+
+.. code-block:: python
+
+ import mlair
+
+ # create your custom MLAir workflow
+ DefaultWorkflow = mlair.DefaultWorkflow()
+ # execute default workflow
+ DefaultWorkflow.run()
+
+The output of running this default workflow will be structured like the following.
+
+.. code-block::
+
+ INFO: mlair started
+ INFO: ExperimentSetup started
+ ...
+ INFO: ExperimentSetup finished after 00:00:01 (hh:mm:ss)
+ INFO: PreProcessing started
+ ...
+ INFO: PreProcessing finished after 00:00:11 (hh:mm:ss)
+ INFO: ModelSetup started
+ ...
+ INFO: ModelSetup finished after 00:00:01 (hh:mm:ss)
+ INFO: Training started
+ ...
+ INFO: Training finished after 00:02:15 (hh:mm:ss)
+ INFO: PostProcessing started
+ ...
+ INFO: PostProcessing finished after 00:01:37 (hh:mm:ss)
+ INFO: mlair finished after 00:04:05 (hh:mm:ss)
+
+Custom Model
+------------
+
+Create your own model to run your personal experiment. To guarantee a proper integration in the MLAir workflow, models
+are restricted to inherit from the :py:`AbstractModelClass`. This will ensure a smooth training and evaluation
+behaviour.
+
+
+How to create a customised model?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* Create a new model class inheriting from :py:`AbstractModelClass`
+
+.. code-block:: python
+
+ from mlair import AbstractModelClass
+ import keras
+
+ class MyCustomisedModel(AbstractModelClass):
+
+ def __init__(self, shape_inputs: list, shape_outputs: list):
+
+ super().__init__(shape_inputs[0], shape_outputs[0])
+
+ # settings
+ self.dropout_rate = 0.1
+ self.activation = keras.layers.PReLU
+
+ # apply to model
+ self.set_model()
+ self.set_compile_options()
+ self.set_custom_objects(loss=self.compile_options['loss'])
+
+* Make sure to add the :py:`super().__init__()` and at least :py:`set_model()` and :py:`set_compile_options()` to your
+ custom init method.
+* The shown model expects a single input and output branch provided in a list. Therefore shapes of input and output are
+ extracted and then provided to the super class initialiser.
+* Some general settings like the dropout rate are set in the init method additionally.
+* If you have custom objects in your model, that are not part of the keras or tensorflow frameworks, you need to add
+ them to custom objects. To do this, call :py:`set_custom_objects` with arbitrarily kwargs. In the shown example, the
+ loss has been added for demonstration only, because we use a build-in loss function. Nonetheless, we always encourage
+ you to add the loss as custom object, to prevent potential errors when loading an already created model instead of
+ training a new one.
+* Now build your model inside :py:`set_model()` by using the instance attributes :py:`self.shape_inputs` and
+ :py:`self.shape_outputs` and storing the model as :py:`self.model`.
+
+.. code-block:: python
+
+ class MyCustomisedModel(AbstractModelClass):
+
+ def set_model(self):
+ x_input = keras.layers.Input(shape=self.shape_inputs)
+ x_in = keras.layers.Conv2D(32, (1, 1), padding='same', name='{}_Conv_1x1'.format("major"))(x_input)
+ x_in = self.activation(name='{}_conv_act'.format("major"))(x_in)
+ x_in = keras.layers.Flatten(name='{}'.format("major"))(x_in)
+ x_in = keras.layers.Dropout(self.dropout_rate, name='{}_Dropout_1'.format("major"))(x_in)
+ x_in = keras.layers.Dense(16, name='{}_Dense_16'.format("major"))(x_in)
+ x_in = self.activation()(x_in)
+ x_in = keras.layers.Dense(self.shape_outputs, name='{}_Dense'.format("major"))(x_in)
+ out_main = self.activation()(x_in)
+ self.model = keras.Model(inputs=x_input, outputs=[out_main])
+
+* Your are free how to design your model. Just make sure to save it in the class attribute model.
+* Additionally, set your custom compile options including the loss definition.
+
+.. code-block:: python
+
+ class MyCustomisedModel(AbstractModelClass):
+
+ def set_compile_options(self):
+ self.initial_lr = 1e-2
+ self.optimizer = keras.optimizers.SGD(lr=self.initial_lr, momentum=0.9)
+ self.lr_decay = mlair.model_modules.keras_extensions.LearningRateDecay(base_lr=self.initial_lr,
+ drop=.94,
+ epochs_drop=10)
+ self.loss = keras.losses.mean_squared_error
+ self.compile_options = {"metrics": ["mse", "mae"]}
+
+* The allocation of the instance parameters :py:`initial_lr`, :py:`optimizer`, and :py:`lr_decay` could be also part of
+ the model class' initialiser. The same applies to :py:`self.loss` and :py:`compile_options`, but we recommend to use
+ the :py:`set_compile_options` method for the definition of parameters, that are related to the compile options.
+* More important is that the compile options are actually saved. There are three ways to achieve this.
+
+ * (1): Set all compile options by parsing a dictionary with all options to :py:`self.compile_options`.
+ * (2): Set all compile options as instance attributes. MLAir will search for these attributes and store them.
+ * (3): Define your compile options partly as dictionary and instance attributes (as shown in this example).
+ * If using (3) and defining the same compile option with different values, MLAir will raise an error.
+
+ Incorrect: (Will raise an error because of a mismatch for the :py:`optimizer` parameter.)
+
+ .. code-block:: python
+
+ def set_compile_options(self):
+ self.optimizer = keras.optimizers.SGD()
+ self.loss = keras.losses.mean_squared_error
+ self.compile_options = {"optimizer" = keras.optimizers.Adam()}
+
+
+Specials for Branched Models
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+* If you have a branched model with multiple outputs, you need either set only a single loss for all branch outputs or
+ provide the same number of loss functions considering the right order.
+
+.. code-block:: python
+
+ class MyCustomisedModel(AbstractModelClass):
+
+ def set_model(self):
+ ...
+ self.model = keras.Model(inputs=x_input, outputs=[out_minor_1, out_minor_2, out_main])
+
+ def set_compile_options(self):
+ self.loss = [keras.losses.mean_absolute_error] + # for out_minor_1
+ [keras.losses.mean_squared_error] + # for out_minor_2
+ [keras.losses.mean_squared_error] # for out_main
+
+
+How to access my customised model?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If the customised model is created, you can easily access the model with
+
+>>> MyCustomisedModel().model
+<your custom model>
+
+The loss is accessible via
+
+>>> MyCustomisedModel().loss
+<your custom loss>
+
+You can treat the instance of your model as instance but also as the model itself. If you call a method, that refers to
+the model instead of the model instance, you can directly apply the command on the instance instead of adding the model
+parameter call.
+
+>>> MyCustomisedModel().model.compile(**kwargs) == MyCustomisedModel().compile(**kwargs)
+True
+
+
+Data Handler
+------------
+
+The basic concept of a data handler is to ensure an appropriate handling of input and target data. This includes the
+loading and preparation of data and their provision in a predefined format. The user is given free rein as to which
+steps the loading and preparation must include. The only constraint is that data is considered as a collection of
+stations. This means that one instance of the data handler is created per station. MLAir then takes over the iteration
+over the collection of stations or distributes the data during the training according to the given batch size. With very
+large data sets, memory problems may occur if all data is loaded and held in main memory. In such a case it is
+recommended to open the data only temporarily. This has no effect on the training itself, as the data is then
+automatically distributed by MLAir.
+
+Interface of a data handler
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A data handler should inherit from the :py:`AbstractDataHandler` class. This class has some key features:
+
+* :py:`cls.requirements()` can be used, to request all available :py:`args` and :py:`kwargs` from MLAir to build the
+ class.
+* :py:`cls.build(*args, **kwargs)` returns in default mode the class itself. This can be modified (=overwritten) to
+ execute some pre-build operations.
+* :py:`self.get_X(upsampling, as_numpy)` should return the input data either as NumPy array or xarray. With the
+ upsamling argument it is possible to implement a feature to weight inputs during training.
+* :py:`self.get_Y(upsampling, as_numpy)` the same but for the target data.
+* :py:`self.transformation(*args, **kwargs)` is a placeholder to execute any desired transformation. This class method
+ is called during the preprocessing stage in the default MLAir workflow. Note that a transformation operation is only
+ estimated on the train data subset and afterwards applied on all data subsets.
+* :py:`self.get_coordinates()` is a placeholder and can be used to return a position for a geographical overview plot.
+
+During the preprocessing stage the following is executed:
+
+1) MLAir requests all required parameters that should be set during Experiment Setup stage by calling
+ :py:`data_handler.requirements()`.
+2) The data handler is build for each station using :py:`data_handler.build(station, **kwargs)` to check if data is
+ available for the given station.
+3) If valid: The build data handler is added to a internal data collection, that collects all contributing data
+ handlers.
+4) MLAir creates subsets for training, validation, and testing. Therefore, a separate data handler for each subset is
+ created using subset parameters (e.g. start and end).
+
+Later on during ModelSetup, Training and PostProcessing, MLAir requests data using :py:`data_handler.get_X()` and
+:py:`data_handler.get_Y()`.
+
+Default Data Handler
+~~~~~~~~~~~~~~~~~~~~
+
+The default data handler accesses data from the TOAR database.
+
+
+Custom Data Handler
+~~~~~~~~~~~~~~~~~~~
+
+* Choose your personal data source, either a web interface or locally available data.
+* Create your custom data handler class by inheriting from :py:`AbstractDataHandler`.
+* Implement the initialiser :py:`__init__(*args, **kwargs)` and make sure to call the super class initialiser as well.
+ After executing this method data should be ready to use. Besides there are no further rules for the initialiser.
+* (optionally) Modify the class method :py:`cls.build(*args, **kwargs)` to calculate pre-build operations. Otherwise the
+ data handler calls the class initialiser. On modification make sure to return the class at the end.
+* (optionally) Add names of required arguments to the :py:`cls._requirements` list. It is not required to add args and
+ kwargs from the initialiser, they are added automatically. Modifying the requirements is only necessary if the build
+ method is modified (see previous bullet).
+* (optionally) Overwrite the base class :py:`self.get_coordinates()` method to return coordinates as dictionary with
+ keys *lon* and *lat*.
+
+.. code-block:: python
+
+ import datetime as dt
+ import numpy as np
+ import pandas as pd
+ import xarray as xr
+
+ from mlair.data_handler import AbstractDataHandler
+
+ class DummyDataHandler(AbstractDataHandler):
+
+ def __init__(self, name, number_of_samples=None):
+ """This data handler takes a name argument and the number of samples to generate. If not provided, a random
+ number between 100 and 150 is set."""
+ super().__init__()
+ self.name = name
+ self.number_of_samples = number_of_samples if number_of_samples is not None else np.random.randint(100, 150)
+ self._X = self.create_X()
+ self._Y = self.create_Y()
+
+ def create_X(self):
+ """Inputs are random numbers between 0 and 10 with shape (no_samples, window=14, variables=5)."""
+ X = np.random.randint(0, 10, size=(self.number_of_samples, 14, 5)) # samples, window, variables
+ datelist = pd.date_range(dt.datetime.today().date(), periods=self.number_of_samples, freq="H").tolist()
+ return xr.DataArray(X, dims=['datetime', 'window', 'variables'], coords={"datetime": datelist,
+ "window": range(14),
+ "variables": range(5)})
+
+ def create_Y(self):
+ """Targets are normal distributed random numbers with shape (no_samples, window=5, variables=1)."""
+ Y = np.round(0.5 * np.random.randn(self.number_of_samples, 5, 1), 1) # samples, window, variables
+ datelist = pd.date_range(dt.datetime.today().date(), periods=self.number_of_samples, freq="H").tolist()
+ return xr.DataArray(Y, dims=['datetime', 'window', 'variables'], coords={"datetime": datelist,
+ "window": range(5),
+ "variables": range(1)})
+
+ def get_X(self, upsampling=False, as_numpy=False):
+ """Upsampling parameter is not used for X."""
+ return np.copy(self._X) if as_numpy is True else self._X
+
+ def get_Y(self, upsampling=False, as_numpy=False):
+ """Upsampling parameter is not used for Y."""
+ return np.copy(self._Y) if as_numpy is True else self._Y
+
+ def __str__(self):
+ return self.name
+
+
+Customised Run Module and Workflow
+----------------------------------
+
+It is possible to create new custom run modules. A custom run module is required to inherit from the base class
+:py:`RunEnvironment` and to hold the constructor method :py:`__init__()`. This method has to execute the module on call.
+In the following example, this is done by using the :py:`_run()` method that is called by the initialiser. It is
+possible to parse arguments to the custom run module as shown.
+
+.. code-block:: python
+
+ import mlair
+ import logging
+
+ class CustomStage(mlair.RunEnvironment):
+ """A custom MLAir stage for demonstration."""
+
+ def __init__(self, test_string):
+ super().__init__() # always call super init method
+ self._run(test_string) # call a class method
+
+ def _run(self, test_string):
+ logging.info("Just running a custom stage.")
+ logging.info("test_string = " + test_string)
+ epochs = self.data_store.get("epochs")
+ logging.info("epochs = " + str(epochs))
+
+
+If a custom run module is defined, it is required to adjust the workflow. For this, you need to load the empty
+:py:`Workflow` class and add each run module that is required. The order of adding modules defines the order of
+execution if running the workflow.
+
+.. code-block:: python
+
+ # create your custom MLAir workflow
+ CustomWorkflow = mlair.Workflow()
+ # provide stages without initialisation
+ CustomWorkflow.add(mlair.ExperimentSetup, epochs=128)
+ # add also keyword arguments for a specific stage
+ CustomWorkflow.add(CustomStage, test_string="Hello World")
+ # finally execute custom workflow in order of adding
+ CustomWorkflow.run()
+
+The output will look like:
+
+.. code-block::
+
+ INFO: mlair started
+ ...
+ INFO: ExperimentSetup finished after 00:00:12 (hh:mm:ss)
+ INFO: CustomStage started
+ INFO: Just running a custom stage.
+ INFO: test_string = Hello World
+ INFO: epochs = 128
+ INFO: CustomStage finished after 00:00:01 (hh:mm:ss)
+ INFO: mlair finished after 00:00:13 (hh:mm:ss)
\ No newline at end of file
diff --git a/docs/_source/get-started.rst b/docs/_source/get-started.rst
index 98a96d43675a0263be5bfc2d452b8af1c2626b60..2e8838fd5b1ac63a7e34e39b7f8bc24d70f9c1b7 100644
--- a/docs/_source/get-started.rst
+++ b/docs/_source/get-started.rst
@@ -1,34 +1,46 @@
-Get started with MLAir
-======================
+
+Getting started with MLAir
+==========================
+
+.. role:: py(code)
+ :language: python
+
Install MLAir
-------------
MLAir is based on several python frameworks. To work properly, you have to install all packages from the
-`requirements.txt` file. Additionally to support the geographical plotting part it is required to install geo
+:py:`requirements.txt` file. Additionally to support the geographical plotting part it is required to install geo
packages built for your operating system. Name names of these package may differ for different systems, we refer
-here to the opensuse / leap OS. The geo plot can be removed from the `plot_list`, in this case there is no need to
+here to the opensuse / leap OS. The geo plot can be removed from the :py:`plot_list`, in this case there is no need to
install the geo packages.
-* (geo) Install **proj** on your machine using the console. E.g. for opensuse / leap `zypper install proj`
+Pre-requirements
+~~~~~~~~~~~~~~~~
+
+* (geo) Install **proj** on your machine using the console. E.g. for opensuse / leap :py:`zypper install proj`
* (geo) A c++ compiler is required for the installation of the program **cartopy**
-* Install all requirements from [`requirements.txt`](https://gitlab.version.fz-juelich.de/toar/machinelearningtools/-/blob/master/requirements.txt)
- preferably in a virtual environment
* (tf) Currently, TensorFlow-1.13 is mentioned in the requirements. We already tested the TensorFlow-1.15 version and couldn't
find any compatibility errors. Please note, that tf-1.13 and 1.15 have two distinct branches each, the default branch
for CPU support, and the "-gpu" branch for GPU support. If the GPU version is installed, MLAir will make use of the GPU
device.
-* Installation of **MLAir**:
- * Either clone MLAir from the [gitlab repository](https://gitlab.version.fz-juelich.de/toar/machinelearningtools.git)
- and use it without installation (beside the requirements)
- * or download the distribution file (?? .whl) and install it via `pip install <??>`. In this case, you can simply
- import MLAir in any python script inside your virtual environment using `import mlair`.
+
+Installation of MLAir
+~~~~~~~~~~~~~~~~~~~~~
+
+* Install all requirements from `requirements.txt <https://gitlab.version.fz-juelich.de/toar/machinelearningtools/-/blob/master/requirements.txt>`_
+ preferably in a virtual environment
+* Either clone MLAir from the `gitlab repository <https://gitlab.version.fz-juelich.de/toar/machinelearningtools.git>`_
+* or download the distribution file (?? .whl) and install it via :py:`pip install <??>`. In this case, you can simply
+ import MLAir in any python script inside your virtual environment using :py:`import mlair`.
How to start with MLAir
-----------------------
-In this section, we show three examples how to work with MLAir.
+In this section, we show three examples how to work with MLAir. Note, that for these examples MLAir was installed using
+the distribution file. In case you are using the git clone it is required to adjust the import path if not directly
+executed inside the source directory of MLAir.
Example 1
~~~~~~~~~
@@ -126,107 +138,3 @@ We can see from the terminal that no training was performed. Analysis is now mad
...
INFO: mlair finished after 00:00:06 (hh:mm:ss)
-
-
-Customised workflows and models
--------------------------------
-
-Custom Workflow
-~~~~~~~~~~~~~~~
-
-MLAir provides a default workflow. If additional steps are to be performed, you have to append custom run modules to
-the workflow.
-
-.. code-block:: python
-
- import mlair
- import logging
-
- class CustomStage(mlair.RunEnvironment):
- """A custom MLAir stage for demonstration."""
-
- def __init__(self, test_string):
- super().__init__() # always call super init method
- self._run(test_string) # call a class method
-
- def _run(self, test_string):
- logging.info("Just running a custom stage.")
- logging.info("test_string = " + test_string)
- epochs = self.data_store.get("epochs")
- logging.info("epochs = " + str(epochs))
-
-
- # create your custom MLAir workflow
- CustomWorkflow = mlair.Workflow()
- # provide stages without initialisation
- CustomWorkflow.add(mlair.ExperimentSetup, epochs=128)
- # add also keyword arguments for a specific stage
- CustomWorkflow.add(CustomStage, test_string="Hello World")
- # finally execute custom workflow in order of adding
- CustomWorkflow.run()
-
-.. code-block::
-
- INFO: mlair started
- ...
- INFO: ExperimentSetup finished after 00:00:12 (hh:mm:ss)
- INFO: CustomStage started
- INFO: Just running a custom stage.
- INFO: test_string = Hello World
- INFO: epochs = 128
- INFO: CustomStage finished after 00:00:01 (hh:mm:ss)
- INFO: mlair finished after 00:00:13 (hh:mm:ss)
-
-Custom Model
-~~~~~~~~~~~~
-
-Each model has to inherit from the abstract model class to ensure a smooth training and evaluation behaviour. It is
-required to implement the set model and set compile options methods. The later has to set the loss at least.
-
-.. code-block:: python
-
- import keras
- from keras.losses import mean_squared_error as mse
- from keras.optimizers import SGD
-
- from mlair.model_modules import AbstractModelClass
-
- class MyLittleModel(AbstractModelClass):
- """
- A customised model with a 1x1 Conv, and 3 Dense layers (32, 16
- window_lead_time). Dropout is used after Conv layer.
- """
- def __init__(self, window_history_size, window_lead_time, channels):
- super().__init__()
- # settings
- self.window_history_size = window_history_size
- self.window_lead_time = window_lead_time
- self.channels = channels
- self.dropout_rate = 0.1
- self.activation = keras.layers.PReLU
- self.lr = 1e-2
- # apply to model
- self.set_model()
- self.set_compile_options()
- self.set_custom_objects(loss=self.compile_options['loss'])
-
- def set_model(self):
- # add 1 to window_size to include current time step t0
- shape = (self.window_history_size + 1, 1, self.channels)
- x_input = keras.layers.Input(shape=shape)
- x_in = keras.layers.Conv2D(32, (1, 1), padding='same')(x_input)
- x_in = self.activation()(x_in)
- x_in = keras.layers.Flatten()(x_in)
- x_in = keras.layers.Dropout(self.dropout_rate)(x_in)
- x_in = keras.layers.Dense(32)(x_in)
- x_in = self.activation()(x_in)
- x_in = keras.layers.Dense(16)(x_in)
- x_in = self.activation()(x_in)
- x_in = keras.layers.Dense(self.window_lead_time)(x_in)
- out = self.activation()(x_in)
- self.model = keras.Model(inputs=x_input, outputs=[out])
-
- def set_compile_options(self):
- self.compile_options = {"optimizer": SGD(lr=self.lr),
- "loss": mse,
- "metrics": ["mse"]}
diff --git a/docs/_source/index.rst b/docs/_source/index.rst
index 341ac58acd62ccc5bcf786580fff1bc193170d62..a60643042eb750b6a21fa0f638797002be7675c4 100644
--- a/docs/_source/index.rst
+++ b/docs/_source/index.rst
@@ -1,17 +1,18 @@
-.. machinelearningtools documentation master file, created by
+.. MLair documentation master file, created by
sphinx-quickstart on Wed Apr 15 14:27:29 2020.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
-Welcome to machinelearningtools's documentation!
+Welcome to MLAir's documentation!
================================================
+
.. toctree::
:maxdepth: 2
:caption: Contents:
get-started
- api
+ customise
Indices and tables
diff --git a/mlair/data_handler/__init__.py b/mlair/data_handler/__init__.py
index 451868b838ab7a0d165942e36b5ec6aa03e42721..6510b336319cfd6a139f70366d0badbb7c1b3587 100644
--- a/mlair/data_handler/__init__.py
+++ b/mlair/data_handler/__init__.py
@@ -11,5 +11,5 @@ __date__ = '2020-04-17'
from .bootstraps import BootStraps
from .iterator import KerasIterator, DataCollection
-from .advanced_data_handler import DefaultDataPreparation, AbstractDataPreparation
-from .data_preparation_neighbors import DataPreparationNeighbors
+from .advanced_data_handler import DefaultDataHandler, AbstractDataHandler
+from .data_preparation_neighbors import DataHandlerNeighbors
diff --git a/mlair/data_handler/advanced_data_handler.py b/mlair/data_handler/advanced_data_handler.py
index 57a9667f2a42575faa02d50e439252738a8dc8bb..bf7defa56709c53e9c11b54baca54efcd105843c 100644
--- a/mlair/data_handler/advanced_data_handler.py
+++ b/mlair/data_handler/advanced_data_handler.py
@@ -17,7 +17,7 @@ import copy
from typing import Union, List, Tuple, Dict
import logging
from functools import reduce
-from mlair.data_handler.station_preparation import StationPrep
+from mlair.data_handler.station_preparation import DataHandlerSingleStation
from mlair.helpers.join import EmptyQueryResult
@@ -49,7 +49,7 @@ class DummyDataSingleStation: # pragma: no cover
return self.name
-class AbstractDataPreparation:
+class AbstractDataHandler:
_requirements = []
@@ -84,14 +84,15 @@ class AbstractDataPreparation:
return self.get_X(upsampling, as_numpy), self.get_Y(upsampling, as_numpy)
def get_coordinates(self) -> Union[None, Dict]:
+ """Return coordinates as dictionary with keys `lon` and `lat`."""
return None
-class DefaultDataPreparation(AbstractDataPreparation):
+class DefaultDataHandler(AbstractDataHandler):
- _requirements = remove_items(inspect.getfullargspec(StationPrep).args, ["self", "station"])
+ _requirements = remove_items(inspect.getfullargspec(DataHandlerSingleStation).args, ["self", "station"])
- def __init__(self, id_class, data_path, min_length=0,
+ def __init__(self, id_class: DataHandlerSingleStation, data_path: str, min_length: int = 0,
extreme_values: num_or_list = None, extremes_on_right_tail_only: bool = False, name_affix=None):
super().__init__()
self.id_class = id_class
@@ -109,9 +110,9 @@ class DefaultDataPreparation(AbstractDataPreparation):
self._store(fresh_store=True)
@classmethod
- def build(cls, station, **kwargs):
+ def build(cls, station: str, **kwargs):
sp_keys = {k: copy.deepcopy(kwargs[k]) for k in cls._requirements if k in kwargs}
- sp = StationPrep(station, **sp_keys)
+ sp = DataHandlerSingleStation(station, **sp_keys)
dp_args = {k: copy.deepcopy(kwargs[k]) for k in cls.own_args("id_class") if k in kwargs}
return cls(sp, **dp_args)
@@ -286,7 +287,7 @@ class DefaultDataPreparation(AbstractDataPreparation):
mean, std = None, None
for station in set_stations:
try:
- sp = StationPrep(station, transformation={"method": method}, **sp_keys)
+ sp = DataHandlerSingleStation(station, transformation={"method": method}, **sp_keys)
mean = sp.mean.copy(deep=True) if mean is None else mean.combine_first(sp.mean)
std = sp.std.copy(deep=True) if std is None else std.combine_first(sp.std)
except (AttributeError, EmptyQueryResult):
@@ -303,23 +304,23 @@ class DefaultDataPreparation(AbstractDataPreparation):
def run_data_prep():
- from .data_preparation_neighbors import DataPreparationNeighbors
- data = DummyDataSingleStation("main_class")
+ from .data_preparation_neighbors import DataHandlerNeighbors
+ data = DummyDataHandler("main_class")
data.get_X()
data.get_Y()
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testdata")
- data_prep = DataPreparationNeighbors(DummyDataSingleStation("main_class"),
- path,
- neighbors=[DummyDataSingleStation("neighbor1"),
- DummyDataSingleStation("neighbor2")],
- extreme_values=[1., 1.2])
+ data_prep = DataHandlerNeighbors(DummyDataHandler("main_class"),
+ path,
+ neighbors=[DummyDataHandler("neighbor1"),
+ DummyDataHandler("neighbor2")],
+ extreme_values=[1., 1.2])
data_prep.get_data(upsampling=False)
def create_data_prep():
- from .data_preparation_neighbors import DataPreparationNeighbors
+ from .data_preparation_neighbors import DataHandlerNeighbors
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testdata")
station_type = None
network = 'UBA'
@@ -329,22 +330,61 @@ def create_data_prep():
interpolation_dim = 'datetime'
window_history_size = 7
window_lead_time = 3
- central_station = StationPrep("DEBW011", path, {'o3': 'dma8eu', 'temp': 'maximum'}, {},station_type, network, sampling, target_dim,
- target_var, interpolation_dim, window_history_size, window_lead_time)
- neighbor1 = StationPrep("DEBW013", path, {'o3': 'dma8eu', 'temp-rea-miub': 'maximum'}, {},station_type, network, sampling, target_dim,
- target_var, interpolation_dim, window_history_size, window_lead_time)
- neighbor2 = StationPrep("DEBW034", path, {'o3': 'dma8eu', 'temp': 'maximum'}, {}, station_type, network, sampling, target_dim,
- target_var, interpolation_dim, window_history_size, window_lead_time)
+ central_station = DataHandlerSingleStation("DEBW011", path, {'o3': 'dma8eu', 'temp': 'maximum'}, {}, station_type, network, sampling, target_dim,
+ target_var, interpolation_dim, window_history_size, window_lead_time)
+ neighbor1 = DataHandlerSingleStation("DEBW013", path, {'o3': 'dma8eu', 'temp-rea-miub': 'maximum'}, {}, station_type, network, sampling, target_dim,
+ target_var, interpolation_dim, window_history_size, window_lead_time)
+ neighbor2 = DataHandlerSingleStation("DEBW034", path, {'o3': 'dma8eu', 'temp': 'maximum'}, {}, station_type, network, sampling, target_dim,
+ target_var, interpolation_dim, window_history_size, window_lead_time)
data_prep = []
- data_prep.append(DataPreparationNeighbors(central_station, path, neighbors=[neighbor1, neighbor2]))
- data_prep.append(DataPreparationNeighbors(neighbor1, path, neighbors=[central_station, neighbor2]))
- data_prep.append(DataPreparationNeighbors(neighbor2, path, neighbors=[neighbor1, central_station]))
+ data_prep.append(DataHandlerNeighbors(central_station, path, neighbors=[neighbor1, neighbor2]))
+ data_prep.append(DataHandlerNeighbors(neighbor1, path, neighbors=[central_station, neighbor2]))
+ data_prep.append(DataHandlerNeighbors(neighbor2, path, neighbors=[neighbor1, central_station]))
return data_prep
+class DummyDataHandler(AbstractDataHandler):
+
+ def __init__(self, name, number_of_samples=None):
+ """This data handler takes a name argument and the number of samples to generate. If not provided, a random
+ number between 100 and 150 is set."""
+ super().__init__()
+ self.name = name
+ self.number_of_samples = number_of_samples if number_of_samples is not None else np.random.randint(100, 150)
+ self._X = self.create_X()
+ self._Y = self.create_Y()
+
+ def create_X(self):
+ """Inputs are random numbers between 0 and 10 with shape (no_samples, window=14, variables=5)."""
+ X = np.random.randint(0, 10, size=(self.number_of_samples, 14, 5)) # samples, window, variables
+ datelist = pd.date_range(dt.datetime.today().date(), periods=self.number_of_samples, freq="H").tolist()
+ return xr.DataArray(X, dims=['datetime', 'window', 'variables'], coords={"datetime": datelist,
+ "window": range(14),
+ "variables": range(5)})
+
+ def create_Y(self):
+ """Targets are normal distributed random numbers with shape (no_samples, window=5, variables=1)."""
+ Y = np.round(0.5 * np.random.randn(self.number_of_samples, 5, 1), 1) # samples, window, variables
+ datelist = pd.date_range(dt.datetime.today().date(), periods=self.number_of_samples, freq="H").tolist()
+ return xr.DataArray(Y, dims=['datetime', 'window', 'variables'], coords={"datetime": datelist,
+ "window": range(5),
+ "variables": range(1)})
+
+ def get_X(self, upsampling=False, as_numpy=False):
+ """Upsampling parameter is not used for X."""
+ return np.copy(self._X) if as_numpy is True else self._X
+
+ def get_Y(self, upsampling=False, as_numpy=False):
+ """Upsampling parameter is not used for Y."""
+ return np.copy(self._Y) if as_numpy is True else self._Y
+
+ def __str__(self):
+ return self.name
+
+
if __name__ == "__main__":
- from mlair.data_handler.station_preparation import StationPrep
+ from mlair.data_handler.station_preparation import DataHandlerSingleStation
from mlair.data_handler.iterator import KerasIterator, DataCollection
data_prep = create_data_prep()
data_collection = DataCollection(data_prep)
diff --git a/mlair/data_handler/bootstraps.py b/mlair/data_handler/bootstraps.py
index 91603b41822b92e28fbd077c502d84707fff746f..f7f5c3c784da0d8a9e05780df19e0b2e2262697a 100644
--- a/mlair/data_handler/bootstraps.py
+++ b/mlair/data_handler/bootstraps.py
@@ -19,7 +19,7 @@ from itertools import chain
import numpy as np
import xarray as xr
-from mlair.data_handler.advanced_data_handler import AbstractDataPreparation
+from mlair.data_handler.advanced_data_handler import AbstractDataHandler
class BootstrapIterator(Iterator):
@@ -82,7 +82,7 @@ class BootStraps(Iterable):
"""
Main class to perform bootstrap operations.
- This class requires a data handler following the definition of the AbstractDataPreparation, the number of bootstraps
+ This class requires a data handler following the definition of the AbstractDataHandler, the number of bootstraps
to create and the dimension along this bootstrapping is performed (default dimension is `variables`).
When iterating on this class, it returns the bootstrapped X, Y and a tuple with (position of variable in X, name of
@@ -91,7 +91,7 @@ class BootStraps(Iterable):
retrieved by calling the .bootstraps() method. Further more, by calling the .get_orig_prediction() this class
imitates according to the set number of bootstraps the original prediction
"""
- def __init__(self, data: AbstractDataPreparation, number_of_bootstraps: int = 10,
+ def __init__(self, data: AbstractDataHandler, number_of_bootstraps: int = 10,
bootstrap_dimension: str = "variables"):
"""
Create iterable class to be ready to iter.
diff --git a/mlair/data_handler/data_preparation_neighbors.py b/mlair/data_handler/data_preparation_neighbors.py
index 0c95b242e1046618403ebb6592407ef8b680e890..37e1922559bbd674706b534fdbc2562e88f66689 100644
--- a/mlair/data_handler/data_preparation_neighbors.py
+++ b/mlair/data_handler/data_preparation_neighbors.py
@@ -4,8 +4,8 @@ __date__ = '2020-07-17'
from mlair.helpers import to_list
-from mlair.data_handler.station_preparation import StationPrep
-from mlair.data_handler.advanced_data_handler import DefaultDataPreparation
+from mlair.data_handler.station_preparation import DataHandlerSingleStation
+from mlair.data_handler.advanced_data_handler import DefaultDataHandler
import os
from typing import Union, List
@@ -14,7 +14,7 @@ number = Union[float, int]
num_or_list = Union[number, List[number]]
-class DataPreparationNeighbors(DefaultDataPreparation):
+class DataHandlerNeighbors(DefaultDataHandler):
def __init__(self, id_class, data_path, neighbors=None, min_length=0,
extreme_values: num_or_list = None, extremes_on_right_tail_only: bool = False):
@@ -25,10 +25,10 @@ class DataPreparationNeighbors(DefaultDataPreparation):
@classmethod
def build(cls, station, **kwargs):
sp_keys = {k: kwargs[k] for k in cls._requirements if k in kwargs}
- sp = StationPrep(station, **sp_keys)
+ sp = DataHandlerSingleStation(station, **sp_keys)
n_list = []
for neighbor in kwargs.get("neighbors", []):
- n_list.append(StationPrep(neighbor, **sp_keys))
+ n_list.append(DataHandlerSingleStation(neighbor, **sp_keys))
else:
kwargs["neighbors"] = n_list if len(n_list) > 0 else None
dp_args = {k: kwargs[k] for k in cls.own_args("id_class") if k in kwargs}
@@ -39,12 +39,12 @@ class DataPreparationNeighbors(DefaultDataPreparation):
def get_coordinates(self, include_neighbors=False):
neighbors = list(map(lambda n: n.get_coordinates(), self.neighbors)) if include_neighbors is True else []
- return [super(DataPreparationNeighbors, self).get_coordinates()].append(neighbors)
+ return [super(DataHandlerNeighbors, self).get_coordinates()].append(neighbors)
if __name__ == "__main__":
- a = DataPreparationNeighbors
+ a = DataHandlerNeighbors
requirements = a.requirements()
kwargs = {"path": os.path.join(os.path.dirname(os.path.abspath(__file__)), "testdata"),
diff --git a/mlair/data_handler/station_preparation.py b/mlair/data_handler/station_preparation.py
index ff8496ab30a3b6392ea2314ef2526c80e0f57591..a278d0df9b34b479223b1aa3a3724d08586f2f87 100644
--- a/mlair/data_handler/station_preparation.py
+++ b/mlair/data_handler/station_preparation.py
@@ -39,7 +39,7 @@ DEFAULT_SAMPLING = "daily"
DEFAULT_INTERPOLATION_METHOD = "linear"
-class AbstractStationPrep(object):
+class AbstractDataHandlerSingleStation(object):
def __init__(self): #, path, station, statistics_per_var, transformation, **kwargs):
pass
@@ -50,7 +50,7 @@ class AbstractStationPrep(object):
raise NotImplementedError
-class StationPrep(AbstractStationPrep):
+class DataHandlerSingleStation(AbstractDataHandlerSingleStation):
def __init__(self, station, data_path, statistics_per_var, station_type=DEFAULT_STATION_TYPE,
network=DEFAULT_NETWORK, sampling=DEFAULT_SAMPLING, target_dim=DEFAULT_TARGET_DIM,
@@ -514,6 +514,59 @@ class StationPrep(AbstractStationPrep):
:param transformation: the transformation dictionary as described above.
:return: updated transformation dictionary
+
+ ## Transformation
+
+ There are two different approaches (called scopes) to transform the data:
+ 1) `station`: transform data for each station independently (somehow like batch normalisation)
+ 1) `data`: transform all data of each station with shared metrics
+
+ Transformation must be set by the `transformation` attribute. If `transformation = None` is given to `ExperimentSetup`,
+ data is not transformed at all. For all other setups, use the following dictionary structure to specify the
+ transformation.
+ ```
+ transformation = {"scope": <...>,
+ "method": <...>,
+ "mean": <...>,
+ "std": <...>}
+ ExperimentSetup(..., transformation=transformation, ...)
+ ```
+
+ ### scopes
+
+ **station**: mean and std are not used
+
+ **data**: either provide already calculated values for mean and std (if required by transformation method), or choose
+ from different calculation schemes, explained in the mean and std section.
+
+ ### supported transformation methods
+ Currently supported methods are:
+ * standardise (default, if method is not given)
+ * centre
+
+ ### mean and std
+ `"mean"="accurate"`: calculate the accurate values of mean and std (depending on method) by using all data. Although,
+ this method is accurate, it may take some time for the calculation. Furthermore, this could potentially lead to memory
+ issue (not explored yet, but could appear for a very big amount of data)
+
+ `"mean"="estimate"`: estimate mean and std (depending on method). For each station, mean and std are calculated and
+ afterwards aggregated using the mean value over all station-wise metrics. This method is less accurate, especially
+ regarding the std calculation but therefore much faster.
+
+ We recommend to use the later method *estimate* because of following reasons:
+ * much faster calculation
+ * real accuracy of mean and std is less important, because it is "just" a transformation / scaling
+ * accuracy of mean is almost as high as in the *accurate* case, because of
+ $\bar{x_{ij}} = \bar{\left(\bar{x_i}\right)_j}$. The only difference is, that in the *estimate* case, each mean is
+ equally weighted for each station independently of the actual data count of the station.
+ * accuracy of std is lower for *estimate* because of $\var{x_{ij}} \ne \bar{\left(\var{x_i}\right)_j}$, but still the mean of all
+ station-wise std is a decent estimate of the true std.
+
+ `"mean"=<value, e.g. xr.DataArray>`: If mean and std are already calculated or shall be set manually, just add the
+ scaling values instead of the calculation method. For method *centre*, std can still be None, but is required for the
+ *standardise* method. **Important**: Format of given values **must** match internal data format of DataPreparation
+ class: `xr.DataArray` with `dims=["variables"]` and one value for each variable.
+
"""
if transformation is None:
return
@@ -681,18 +734,18 @@ if __name__ == "__main__":
# dp = AbstractDataPrep('data/', 'dummy', 'DEBW107', ['o3', 'temp'], statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'})
# print(dp)
statistics_per_var = {'o3': 'dma8eu', 'temp-rea-miub': 'maximum'}
- sp = StationPrep(data_path='/home/felix/PycharmProjects/mlt_new/data/', station='DEBY122',
- statistics_per_var=statistics_per_var, station_type='background',
- network='UBA', sampling='daily', target_dim='variables', target_var='o3',
- time_dim='datetime', window_history_size=7, window_lead_time=3,
- interpolation_limit=0
- ) # transformation={'method': 'standardise'})
+ sp = DataHandlerSingleStation(data_path='/home/felix/PycharmProjects/mlt_new/data/', station='DEBY122',
+ statistics_per_var=statistics_per_var, station_type='background',
+ network='UBA', sampling='daily', target_dim='variables', target_var='o3',
+ time_dim='datetime', window_history_size=7, window_lead_time=3,
+ interpolation_limit=0
+ ) # transformation={'method': 'standardise'})
# sp.set_transformation({'method': 'standardise', 'mean': sp.mean+2, 'std': sp.std+1})
- sp2 = StationPrep(data_path='/home/felix/PycharmProjects/mlt_new/data/', station='DEBY122',
- statistics_per_var=statistics_per_var, station_type='background',
- network='UBA', sampling='daily', target_dim='variables', target_var='o3',
- time_dim='datetime', window_history_size=7, window_lead_time=3,
- transformation={'method': 'standardise'})
+ sp2 = DataHandlerSingleStation(data_path='/home/felix/PycharmProjects/mlt_new/data/', station='DEBY122',
+ statistics_per_var=statistics_per_var, station_type='background',
+ network='UBA', sampling='daily', target_dim='variables', target_var='o3',
+ time_dim='datetime', window_history_size=7, window_lead_time=3,
+ transformation={'method': 'standardise'})
sp2.transform(inverse=True)
sp.get_X()
sp.get_Y()
diff --git a/mlair/model_modules/model_class.py b/mlair/model_modules/model_class.py
index bba1d8bd7b95b6aba1a6390a6b0eba384e6780a7..0e69d22012a592b30c6ffdf9ed6082c47a291f90 100644
--- a/mlair/model_modules/model_class.py
+++ b/mlair/model_modules/model_class.py
@@ -351,9 +351,8 @@ class AbstractModelClass(ABC):
class MyLittleModel(AbstractModelClass):
"""
- A customised model with a 1x1 Conv, and 4 Dense layers (64, 32, 16, window_lead_time), where the last layer is the
- output layer depending on the window_lead_time parameter. Dropout is used between the Convolution and the first
- Dense layer.
+ A customised model 4 Dense layers (64, 32, 16, window_lead_time), where the last layer is the output layer depending
+ on the window_lead_time parameter.
"""
def __init__(self, shape_inputs: list, shape_outputs: list):
@@ -382,13 +381,8 @@ class MyLittleModel(AbstractModelClass):
"""
Build the model.
"""
-
- # add 1 to window_size to include current time step t0
x_input = keras.layers.Input(shape=self.shape_inputs)
- x_in = keras.layers.Conv2D(32, (1, 1), padding='same', name='{}_Conv_1x1'.format("major"))(x_input)
- x_in = self.activation(name='{}_conv_act'.format("major"))(x_in)
- x_in = keras.layers.Flatten(name='{}'.format("major"))(x_in)
- x_in = keras.layers.Dropout(self.dropout_rate, name='{}_Dropout_1'.format("major"))(x_in)
+ x_in = keras.layers.Flatten(name='{}'.format("major"))(x_input)
x_in = keras.layers.Dense(64, name='{}_Dense_64'.format("major"))(x_in)
x_in = self.activation()(x_in)
x_in = keras.layers.Dense(32, name='{}_Dense_32'.format("major"))(x_in)
diff --git a/mlair/plotting/postprocessing_plotting.py b/mlair/plotting/postprocessing_plotting.py
index 5cc449aac88ebab58689656820769fe7751f6098..675e5ade587011a9ac835e9afb45f89173bc7653 100644
--- a/mlair/plotting/postprocessing_plotting.py
+++ b/mlair/plotting/postprocessing_plotting.py
@@ -786,8 +786,8 @@ class PlotTimeSeries:
def _plot(self, plot_folder):
pdf_pages = self._create_pdf_pages(plot_folder)
for pos, station in enumerate(self._stations):
- start, end = self._get_time_range(self._load_data(self._stations[0]))
data = self._load_data(station)
+ start, end = self._get_time_range(data)
fig, axes, factor = self._create_subplots(start, end)
nan_list = []
for i_year in range(end - start + 1):
diff --git a/mlair/run_modules/experiment_setup.py b/mlair/run_modules/experiment_setup.py
index 407465ad4cd99b85c3c5b37eb2aef6e9e71c6424..51e710c2d08d759883080406fc988847de832aca 100644
--- a/mlair/run_modules/experiment_setup.py
+++ b/mlair/run_modules/experiment_setup.py
@@ -18,7 +18,7 @@ from mlair.configuration.defaults import DEFAULT_STATIONS, DEFAULT_VAR_ALL_DICT,
DEFAULT_VAL_MIN_LENGTH, DEFAULT_TEST_START, DEFAULT_TEST_END, DEFAULT_TEST_MIN_LENGTH, DEFAULT_TRAIN_VAL_MIN_LENGTH, \
DEFAULT_USE_ALL_STATIONS_ON_ALL_DATA_SETS, DEFAULT_EVALUATE_BOOTSTRAPS, DEFAULT_CREATE_NEW_BOOTSTRAPS, \
DEFAULT_NUMBER_OF_BOOTSTRAPS, DEFAULT_PLOT_LIST
-from mlair.data_handler.advanced_data_handler import DefaultDataPreparation
+from mlair.data_handler.advanced_data_handler import DefaultDataHandler
from mlair.run_modules.run_environment import RunEnvironment
from mlair.model_modules.model_class import MyLittleModel as VanillaModel
@@ -221,7 +221,7 @@ class ExperimentSetup(RunEnvironment):
train_min_length=None, val_min_length=None, test_min_length=None, extreme_values: list = None,
extremes_on_right_tail_only: bool = None, evaluate_bootstraps=None, plot_list=None, number_of_bootstraps=None,
create_new_bootstraps=None, data_path: str = None, batch_path: str = None, login_nodes=None,
- hpc_hosts=None, model=None, batch_size=None, epochs=None, data_preparation=None, **kwargs):
+ hpc_hosts=None, model=None, batch_size=None, epochs=None, data_handler=None, **kwargs):
# create run framework
super().__init__()
@@ -290,7 +290,7 @@ class ExperimentSetup(RunEnvironment):
self._set_param("sampling", sampling)
self._set_param("transformation", transformation, default=DEFAULT_TRANSFORMATION)
self._set_param("transformation", None, scope="preprocessing")
- self._set_param("data_preparation", data_preparation, default=DefaultDataPreparation)
+ self._set_param("data_handler", data_handler, default=DefaultDataHandler)
# target
self._set_param("target_var", target_var, default=DEFAULT_TARGET_VAR)
diff --git a/mlair/run_modules/post_processing.py b/mlair/run_modules/post_processing.py
index d4f409ec503ba0ae37bdd1d1bec4b0207eec453c..b4af7a754335e8da6d29870b1a0c4152d7dc9af5 100644
--- a/mlair/run_modules/post_processing.py
+++ b/mlair/run_modules/post_processing.py
@@ -81,16 +81,12 @@ class PostProcessing(RunEnvironment):
def _run(self):
# ols model
- with TimeTracking():
- self.train_ols_model()
- logging.info("take a look on the next reported time measure. If this increases a lot, one should think to "
- "skip train_ols_model() whenever it is possible to save time.")
+ self.train_ols_model()
# forecasts
- with TimeTracking():
- self.make_prediction()
- logging.info("take a look on the next reported time measure. If this increases a lot, one should think to "
- "skip make_prediction() whenever it is possible to save time.")
+ self.make_prediction()
+
+ # skill scores on test data
self.calculate_test_score()
# bootstraps
diff --git a/mlair/run_modules/pre_processing.py b/mlair/run_modules/pre_processing.py
index b4185df2f6699cb20ac96e32661433e7a6164abc..ed972896e7a39b0b56df23dbc8a8d1ae64fb4183 100644
--- a/mlair/run_modules/pre_processing.py
+++ b/mlair/run_modules/pre_processing.py
@@ -10,7 +10,7 @@ from typing import Tuple
import numpy as np
import pandas as pd
-from mlair.data_handler import DataCollection
+from mlair.data_handler import DataCollection, AbstractDataHandler
from mlair.helpers import TimeTracking
from mlair.configuration import path_config
from mlair.helpers.join import EmptyQueryResult
@@ -55,8 +55,8 @@ class PreProcessing(RunEnvironment):
def _run(self):
stations = self.data_store.get("stations")
- data_preparation = self.data_store.get("data_preparation")
- _, valid_stations = self.validate_station(data_preparation, stations, "preprocessing", overwrite_local_data=True)
+ data_handler = self.data_store.get("data_handler")
+ _, valid_stations = self.validate_station(data_handler, stations, "preprocessing", overwrite_local_data=True)
if len(valid_stations) == 0:
raise ValueError("Couldn't find any valid data according to given parameters. Abort experiment run.")
self.data_store.set("stations", valid_stations)
@@ -187,12 +187,12 @@ class PreProcessing(RunEnvironment):
set_stations = stations[index_list]
logging.debug(f"{set_name.capitalize()} stations (len={len(set_stations)}): {set_stations}")
# create set data_collection and store
- data_preparation = self.data_store.get("data_preparation")
- collection, valid_stations = self.validate_station(data_preparation, set_stations, set_name)
+ data_handler = self.data_store.get("data_handler")
+ collection, valid_stations = self.validate_station(data_handler, set_stations, set_name)
self.data_store.set("stations", valid_stations, scope=set_name)
self.data_store.set("data_collection", collection, scope=set_name)
- def validate_station(self, data_preparation, set_stations, set_name=None, overwrite_local_data=False):
+ def validate_station(self, data_handler: AbstractDataHandler, set_stations, set_name=None, overwrite_local_data=False):
"""
Check if all given stations in `all_stations` are valid.
@@ -212,14 +212,14 @@ class PreProcessing(RunEnvironment):
logging.info(f"check valid stations started{' (%s)' % (set_name if set_name is not None else 'all')}")
# calculate transformation using train data
if set_name == "train":
- self.transformation(data_preparation, set_stations)
+ self.transformation(data_handler, set_stations)
# start station check
collection = DataCollection()
valid_stations = []
- kwargs = self.data_store.create_args_dict(data_preparation.requirements(), scope=set_name)
+ kwargs = self.data_store.create_args_dict(data_handler.requirements(), scope=set_name)
for station in set_stations:
try:
- dp = data_preparation.build(station, name_affix=set_name, **kwargs)
+ dp = data_handler.build(station, name_affix=set_name, **kwargs)
collection.add(dp)
valid_stations.append(station)
except (AttributeError, EmptyQueryResult):
@@ -228,10 +228,10 @@ class PreProcessing(RunEnvironment):
f"{len(set_stations)} valid stations.")
return collection, valid_stations
- def transformation(self, data_preparation, stations):
- if hasattr(data_preparation, "transformation"):
- kwargs = self.data_store.create_args_dict(data_preparation.requirements(), scope="train")
- transformation_dict = data_preparation.transformation(stations, **kwargs)
+ def transformation(self, data_handler: AbstractDataHandler, stations):
+ if hasattr(data_handler, "transformation"):
+ kwargs = self.data_store.create_args_dict(data_handler.requirements(), scope="train")
+ transformation_dict = data_handler.transformation(stations, **kwargs)
if transformation_dict is not None:
self.data_store.set("transformation", transformation_dict)
diff --git a/mlair/run_script.py b/mlair/run_script.py
index 00a28f686bf392f76787b56a48790999e9fa5c05..6dea98ba9c67cfd9cf0ba07d78733a7b7d75909f 100644
--- a/mlair/run_script.py
+++ b/mlair/run_script.py
@@ -27,7 +27,7 @@ def run(stations=None,
model=None,
batch_size=None,
epochs=None,
- data_preparation=None,
+ data_handler=None,
**kwargs):
params = inspect.getfullargspec(DefaultWorkflow).args
diff --git a/requirements.txt b/requirements.txt
index 7da29a05b748531fd4ec327ff17f432ff1ecaabb..e2d8f5bc47e3ab1365814259b313b1a07e54fff3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -65,3 +65,5 @@ wcwidth==0.1.8
Werkzeug==1.0.0
xarray==0.15.0
zipp==3.1.0
+
+setuptools~=49.6.0
\ No newline at end of file
diff --git a/test/test_run_modules/test_pre_processing.py b/test/test_run_modules/test_pre_processing.py
index 97e73204068d334590ee98271080acddf29dfc5f..bdb8fdabff67ad894275c805522b9df4cf167011 100644
--- a/test/test_run_modules/test_pre_processing.py
+++ b/test/test_run_modules/test_pre_processing.py
@@ -2,7 +2,7 @@ import logging
import pytest
-from mlair.data_handler import DefaultDataPreparation, DataCollection, AbstractDataPreparation
+from mlair.data_handler import DefaultDataHandler, DataCollection, AbstractDataHandler
from mlair.helpers.datastore import NameNotFoundInScope
from mlair.helpers import PyTestRegex
from mlair.run_modules.experiment_setup import ExperimentSetup
@@ -28,7 +28,7 @@ class TestPreProcessing:
def obj_with_exp_setup(self):
ExperimentSetup(stations=['DEBW107', 'DEBY081', 'DEBW013', 'DEBW076', 'DEBW087', 'DEBW001'],
statistics_per_var={'o3': 'dma8eu', 'temp': 'maximum'}, station_type="background",
- data_preparation=DefaultDataPreparation)
+ data_handler=DefaultDataHandler)
pre = object.__new__(PreProcessing)
super(PreProcessing, pre).__init__()
yield pre
@@ -90,7 +90,7 @@ class TestPreProcessing:
pre = obj_with_exp_setup
caplog.set_level(logging.INFO)
stations = pre.data_store.get("stations", "general")
- data_preparation = pre.data_store.get("data_preparation")
+ data_preparation = pre.data_store.get("data_handler")
collection, valid_stations = pre.validate_station(data_preparation, stations, set_name=name)
assert isinstance(collection, DataCollection)
assert len(valid_stations) < len(stations)
@@ -110,7 +110,7 @@ class TestPreProcessing:
def test_transformation(self):
pre = object.__new__(PreProcessing)
- data_preparation = AbstractDataPreparation
+ data_preparation = AbstractDataHandler
stations = ['DEBW107', 'DEBY081']
assert pre.transformation(data_preparation, stations) is None
class data_preparation_no_trans: pass
diff --git a/test/test_run_modules/test_training.py b/test/test_run_modules/test_training.py
index 1fec8f4e56e2925bff0bc4af859dac1fe5fbb2b6..c5f1ba9d407340974946398a37dd25234d3cbd78 100644
--- a/test/test_run_modules/test_training.py
+++ b/test/test_run_modules/test_training.py
@@ -9,7 +9,7 @@ import mock
import pytest
from keras.callbacks import History
-from mlair.data_handler import DataCollection, KerasIterator, DefaultDataPreparation
+from mlair.data_handler import DataCollection, KerasIterator, DefaultDataHandler
from mlair.helpers import PyTestRegex
from mlair.model_modules.flatten import flatten_tail
from mlair.model_modules.inception_model import InceptionModelBase
@@ -125,12 +125,12 @@ class TestTraining:
@pytest.fixture
def data_collection(self, path, window_history_size, window_lead_time, statistics_per_var):
- data_prep = DefaultDataPreparation.build(['DEBW107'], data_path=os.path.join(os.path.dirname(__file__), 'data'),
- statistics_per_var=statistics_per_var, station_type="background",
- network="AIRBASE", sampling="daily", target_dim="variables",
- target_var="o3", time_dim="datetime",
- window_history_size=window_history_size,
- window_lead_time=window_lead_time, name_affix="train")
+ data_prep = DefaultDataHandler.build(['DEBW107'], data_path=os.path.join(os.path.dirname(__file__), 'data'),
+ statistics_per_var=statistics_per_var, station_type="background",
+ network="AIRBASE", sampling="daily", target_dim="variables",
+ target_var="o3", time_dim="datetime",
+ window_history_size=window_history_size,
+ window_lead_time=window_lead_time, name_affix="train")
return DataCollection([data_prep])
@pytest.fixture