diff --git a/mlair/reference_data_handler/IntelliO3v1Reference.py b/mlair/reference_data_handler/IntelliO3v1Reference.py new file mode 100644 index 0000000000000000000000000000000000000000..ca2f025cfe47e4bbdf7e81918ab86701f27a855e --- /dev/null +++ b/mlair/reference_data_handler/IntelliO3v1Reference.py @@ -0,0 +1,127 @@ +""" +Extract forecasts from intelliO3 and store them for MLAir + +""" + +__author__ = "Felix Kleinert" +__date__ = "2021-01-29" + +import os +import xarray as xr +import wget +import sys +import shutil + +from mlair.configuration.path_config import check_path_and_create +from mlair.reference_data_handler.abstract_reference_data_handler import AbstractReferenceModel + + +class AbstractReferenceb2share(AbstractReferenceModel): + """ + Abstract class for reference models located on b2share (eudat or fz-juelich) + See also https://github.com/EUDAT-Training/B2SHARE-Training/blob/master/api/01_Retrieve_existing_record.md + + """ + def __init__(self, b2share_hosturl: str, b2share_bucket: str, b2share_key: str): + super().__init__() + self.b2share_hosturl = b2share_hosturl + self.b2share_bucket = b2share_bucket + self.b2share_key = b2share_key + + @property + def b2share_url(self): + return f"{self.b2share_hosturl}/api/files/{self.b2share_bucket}" + + def bar_custom(self, current, total, width=80): + progress_message = f"Downloading {self.b2share_key}: {round(current / total * 100)}% [{current} / {total}] bytes" + sys.stdout.write("\r" + progress_message) + sys.stdout.flush() + + def download_from_b2share(self, tmp_download_path: str): + check_path_and_create(tmp_download_path) + wget.download(f"{self.b2share_url}/{self.b2share_key}", + out=f"{tmp_download_path}{self.b2share_key}", + bar=self.bar_custom) + + +class IntelliO3Reference(AbstractReferenceb2share): + """ + Reference handler that extracts IntelliO3-ts v1.0 forecasts (Kleinert, 2021). + + IntelliO3 forecasts can be used as a competitive model within MLAir. Downloads the IntelliO3 tar-ball and extracts + the forecasts. + + Kleinert, F., Leufen, L. H., and Schultz, M. G.: IntelliO3-ts v1.0: a neural network approach to predict + near-surface ozone concentrations in Germany, Geosci. Model Dev., 14, 1–25, + https://doi.org/10.5194/gmd-14-1-2021, 2021. + """ + + def __init__(self, ref_name: str, ref_store_path: str = None): + """ + :param ref_name: Desired Name of reference forecast + :type ref_name: str + :param ref_store_path: Path to store reference forecasts + :type ref_store_path: str + """ + super().__init__(b2share_hosturl="https://b2share.eudat.eu", + b2share_bucket="0cae9db2-f388-4136-8d28-9d9c5665d641", + b2share_key="IntelliO3-ts.tar.gz", + ) + self.ref_name = ref_name + if ref_store_path is None: + ref_store_path = f"{self.ref_name}/" + self.ref_store_path = ref_store_path + self.tmp_extract_path = "tmp_download/" + self.orig_forecast_path = "IntelliO3-ts/IntelliO3-ts_network/forecasts/" + self.file_pattern = "forecasts_DE?????_test.nc" + + def untar_forecasts(self): + """ + Extracts IntelliO3 forecasts from tar-ball. + """ + cmd = f"tar -xf {self.tmp_extract_path}{self.b2share_key} --directory {self.tmp_extract_path} --wildcards --no-anchored '{self.orig_forecast_path}{self.file_pattern}'" + os.system(cmd) + + def file_list(self): + """ + :return: base dir of tmp path and list of forecast files + :rtype: tuple(str, list(str)) + """ + for root, dirs, file_names in os.walk(self.tmp_extract_path+self.orig_forecast_path): + pass + return root, file_names + + def read_and_drop(self, sel_coords: dict = None): + """ + Reads original forecast files, renames coord type and store forecasts as NetCdf4 files + :param sel_coords: + :type sel_coords: + """ + if sel_coords is None: + sel_coords = {'type': 'CNN'} + in_path, files = self.file_list() + check_path_and_create(self.ref_store_path) + for infile in files: + data = xr.open_dataarray(f"{in_path}{infile}") + data = data.sel(**sel_coords) + data.coords['type'] = (self.ref_name) + data.to_netcdf(f"{self.ref_store_path}{infile}") + + def make_reference_available_locally(self): + """ + + :return: + :rtype: + """ + if not self.is_reference_available_locally(self.ref_store_path): + if not os.path.exists(self.tmp_extract_path+self.b2share_key): + self.download_from_b2share(tmp_download_path=self.tmp_extract_path) + self.untar_forecasts() + self.read_and_drop() + shutil.rmtree(self.tmp_extract_path) + + +if __name__ == '__main__': + io3 = IntelliO3Reference('IntelliO3-ts') + io3.make_reference_available_locally() + diff --git a/mlair/reference_data_handler/__init__.py b/mlair/reference_data_handler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/mlair/reference_data_handler/abstract_reference_data_handler.py b/mlair/reference_data_handler/abstract_reference_data_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..e9d129a722b1e90c07012d9c942f6f70e5d0d69a --- /dev/null +++ b/mlair/reference_data_handler/abstract_reference_data_handler.py @@ -0,0 +1,32 @@ +__author__ = "Felix Kleinert" +__date__ = "2021-01-29" + +import os +from abc import ABC + + +class AbstractReferenceModel(ABC): + """ + Abstract reference model. All classes providing some reference or competitor models must inherent from this class. + """ + def __init__(self, *args, **kwargs): + pass + + def make_reference_available_locally(self): + raise NotImplementedError + + @staticmethod + def is_reference_available_locally(reference_path) -> bool: + """ + Checks if reference is available locally + """ + + try: + if os.listdir(reference_path): + res = True + else: + res = False + except FileNotFoundError: + res = False + return res +