add first implementation to download forecasts from INtelliO3. Related to #131

mlair/reference_data_handler/IntelliO3v1Reference.py

+"""
+Extract forecasts from intelliO3 and store them for MLAir
+
+"""
+
+__author__ = "Felix Kleinert"
+__date__ = "2021-01-29"
+
+import os
+import xarray as xr
+import wget
+import sys
+import shutil
+
+from mlair.configuration.path_config import check_path_and_create
+from mlair.reference_data_handler.abstract_reference_data_handler import AbstractReferenceModel
+
+
+class AbstractReferenceb2share(AbstractReferenceModel):
+    """
+    Abstract class for reference models located on b2share (eudat or fz-juelich)
+    See also https://github.com/EUDAT-Training/B2SHARE-Training/blob/master/api/01_Retrieve_existing_record.md
+
+    """
+    def __init__(self, b2share_hosturl: str, b2share_bucket: str, b2share_key: str):
+        super().__init__()
+        self.b2share_hosturl = b2share_hosturl
+        self.b2share_bucket = b2share_bucket
+        self.b2share_key = b2share_key
+
+    @property
+    def b2share_url(self):
+        return f"{self.b2share_hosturl}/api/files/{self.b2share_bucket}"
+
+    def bar_custom(self, current, total, width=80):
+        progress_message = f"Downloading {self.b2share_key}: {round(current / total * 100)}% [{current} / {total}] bytes"
+        sys.stdout.write("\r" + progress_message)
+        sys.stdout.flush()
+
+    def download_from_b2share(self, tmp_download_path: str):
+        check_path_and_create(tmp_download_path)
+        wget.download(f"{self.b2share_url}/{self.b2share_key}",
+                      out=f"{tmp_download_path}{self.b2share_key}",
+                      bar=self.bar_custom)
+
+
+class IntelliO3Reference(AbstractReferenceb2share):
+    """
+    Reference handler that extracts IntelliO3-ts v1.0 forecasts (Kleinert, 2021).
+
+    IntelliO3 forecasts can be used as a competitive model within MLAir. Downloads the IntelliO3 tar-ball and extracts
+    the forecasts.
+
+    Kleinert, F., Leufen, L. H., and Schultz, M. G.: IntelliO3-ts v1.0: a neural network approach to predict
+    near-surface ozone concentrations in Germany, Geosci. Model Dev., 14, 1–25,
+    https://doi.org/10.5194/gmd-14-1-2021, 2021.
+    """
+
+    def __init__(self, ref_name: str, ref_store_path: str = None):
+        """
+        :param ref_name: Desired Name of reference forecast
+        :type ref_name: str
+        :param ref_store_path: Path to store reference forecasts
+        :type ref_store_path: str
+        """
+        super().__init__(b2share_hosturl="https://b2share.eudat.eu",
+                         b2share_bucket="0cae9db2-f388-4136-8d28-9d9c5665d641",
+                         b2share_key="IntelliO3-ts.tar.gz",
+                         )
+        self.ref_name = ref_name
+        if ref_store_path is None:
+            ref_store_path = f"{self.ref_name}/"
+        self.ref_store_path = ref_store_path
+        self.tmp_extract_path = "tmp_download/"
+        self.orig_forecast_path = "IntelliO3-ts/IntelliO3-ts_network/forecasts/"
+        self.file_pattern = "forecasts_DE?????_test.nc"
+
+    def untar_forecasts(self):
+        """
+        Extracts IntelliO3 forecasts from tar-ball.
+        """
+        cmd = f"tar -xf {self.tmp_extract_path}{self.b2share_key} --directory {self.tmp_extract_path} --wildcards --no-anchored '{self.orig_forecast_path}{self.file_pattern}'"
+        os.system(cmd)
+
+    def file_list(self):
+        """
+        :return: base dir of tmp path and list of forecast files
+        :rtype: tuple(str, list(str))
+        """
+        for root, dirs, file_names in os.walk(self.tmp_extract_path+self.orig_forecast_path):
+            pass
+        return root, file_names
+
+    def read_and_drop(self, sel_coords: dict = None):
+        """
+        Reads original forecast files, renames coord type and store forecasts as NetCdf4 files
+        :param sel_coords:
+        :type sel_coords:
+        """
+        if sel_coords is None:
+            sel_coords = {'type': 'CNN'}
+        in_path, files = self.file_list()
+        check_path_and_create(self.ref_store_path)
+        for infile in files:
+            data = xr.open_dataarray(f"{in_path}{infile}")
+            data = data.sel(**sel_coords)
+            data.coords['type'] = (self.ref_name)
+            data.to_netcdf(f"{self.ref_store_path}{infile}")
+
+    def make_reference_available_locally(self):
+        """
+
+        :return:
+        :rtype:
+        """
+        if not self.is_reference_available_locally(self.ref_store_path):
+            if not os.path.exists(self.tmp_extract_path+self.b2share_key):
+                self.download_from_b2share(tmp_download_path=self.tmp_extract_path)
+            self.untar_forecasts()
+            self.read_and_drop()
+            shutil.rmtree(self.tmp_extract_path)
+
+
+if __name__ == '__main__':
+    io3 = IntelliO3Reference('IntelliO3-ts')
+    io3.make_reference_available_locally()
+

mlair/reference_data_handler/abstract_reference_data_handler.py

+__author__ = "Felix Kleinert"
+__date__ = "2021-01-29"
+
+import os
+from abc import ABC
+
+
+class AbstractReferenceModel(ABC):
+    """
+    Abstract reference model. All classes providing some reference or competitor models must inherent from this class.
+    """
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def make_reference_available_locally(self):
+        raise NotImplementedError
+
+    @staticmethod
+    def is_reference_available_locally(reference_path) -> bool:
+        """
+        Checks if reference is available locally
+        """
+
+        try:
+            if os.listdir(reference_path):
+                res = True
+            else:
+                res = False
+        except FileNotFoundError:
+            res = False
+        return res
+