diff --git a/mlair/helpers/data_sources/join.py b/mlair/helpers/data_sources/join.py index df3b358ed187d4c6460f3d40b7ec20599526b86d..a978b2712a83b21f3c1256b2bf0826da63bdda3a 100644 --- a/mlair/helpers/data_sources/join.py +++ b/mlair/helpers/data_sources/join.py @@ -80,7 +80,7 @@ def download_join(station_name: Union[str, List[str]], stat_var: dict, station_t data = correct_data_format(data) # correct namespace of statistics - stat = _correct_stat_name(stat_var[var]) + stat = toar_data.correct_stat_name(stat_var[var]) # store data in pandas dataframe df = _save_to_pandas(df, data, stat, var) @@ -344,20 +344,6 @@ def _save_to_pandas(df: Union[pd.DataFrame, None], data: dict, stat: str, var: s return df -def _correct_stat_name(stat: str) -> str: - """ - Map given statistic name to new namespace defined by mapping dict. - - Return given name stat if not element of mapping namespace. - - :param stat: namespace from JOIN server - - :return: stat mapped to local namespace - """ - mapping = {'average_values': 'mean', 'maximum': 'max', 'minimum': 'min'} - return mapping.get(stat, stat) - - def _lower_list(args: List[str]) -> Iterator[str]: """ Lower all elements of given list. diff --git a/mlair/helpers/data_sources/toar_data.py b/mlair/helpers/data_sources/toar_data.py index c2aecfd3b72b6abecff8f85892427921c1081e6c..27522855cbe0f3c6f0b78d1598709a694fc7b862 100644 --- a/mlair/helpers/data_sources/toar_data.py +++ b/mlair/helpers/data_sources/toar_data.py @@ -112,3 +112,17 @@ def merge_toar_join(df_toar, df_join, sampling): full_data = df_toar.reindex(full_time) full_data.update(df_join, overwrite=False) return full_data + + +def correct_stat_name(stat: str) -> str: + """ + Map given statistic name to new namespace defined by mapping dict. + + Return given name stat if not element of mapping namespace. + + :param stat: namespace from JOIN server + + :return: stat mapped to local namespace + """ + mapping = {'average_values': 'mean', 'maximum': 'max', 'minimum': 'min'} + return mapping.get(stat, stat) diff --git a/mlair/helpers/data_sources/toar_data_v2.py b/mlair/helpers/data_sources/toar_data_v2.py index a592d2d16e6873ae8dfcef9b27698aaeca35ccb4..0fa53a7eb23f11675eeef2c12a7d5dceec3c38ac 100644 --- a/mlair/helpers/data_sources/toar_data_v2.py +++ b/mlair/helpers/data_sources/toar_data_v2.py @@ -13,8 +13,7 @@ from timezonefinder import TimezoneFinder from mlair.configuration.toar_data_v2_settings import toar_data_v2_settings from mlair.helpers import to_list -from mlair.helpers.data_sources.toar_data import EmptyQueryResult, get_data -from mlair.helpers.data_sources import join +from mlair.helpers.data_sources.toar_data import EmptyQueryResult, get_data, correct_stat_name str_or_none = Union[str, None] @@ -174,8 +173,9 @@ def load_timeseries_data(timeseries_meta, url_base, opts, headers, sampling): res = get_data(opts, headers, as_json=False) data = pd.read_csv(StringIO(res), comment="#", index_col="datetime", parse_dates=True, infer_datetime_format=True) - data = data[join._correct_stat_name(opts.get("names", "value"))].rename(meta["variable"]["name"]) - coll.append(data) + if len(data.index) > 0: + data = data[correct_stat_name(opts.get("names", "value"))].rename(meta["variable"]["name"]) + coll.append(data) return coll diff --git a/test/test_helpers/test_data_sources/test_join.py b/test/test_helpers/test_data_sources/test_join.py index 0a9715f5dba725dc7f8fa184021b925857c707eb..f9b12f5a7ff20e898695de0a0f035bed023674f2 100644 --- a/test/test_helpers/test_data_sources/test_join.py +++ b/test/test_helpers/test_data_sources/test_join.py @@ -3,7 +3,7 @@ from typing import Iterable import pytest from mlair.helpers.data_sources.join import * -from mlair.helpers.data_sources.join import _save_to_pandas, _correct_stat_name, _lower_list, _select_distinct_series, \ +from mlair.helpers.data_sources.join import _save_to_pandas, _lower_list, _select_distinct_series, \ _select_distinct_data_origin, _select_distinct_network from mlair.configuration.join_settings import join_settings from mlair.helpers.testing import check_nested_equality @@ -334,18 +334,6 @@ class TestSaveToPandas: assert pd.testing.assert_frame_equal(create_df, _save_to_pandas(None, data, 'mean', 'cloudcover')) is None -class TestCorrectStatName: - - def test_nothing_to_do(self): - assert _correct_stat_name("dma8eu") == "dma8eu" - assert _correct_stat_name("max") == "max" - - def test_correct_string(self): - assert _correct_stat_name("maximum") == "max" - assert _correct_stat_name("minimum") == "min" - assert _correct_stat_name("average_values") == "mean" - - class TestLowerList: def test_string_lowering(self): diff --git a/test/test_helpers/test_data_sources/test_toar_data.py b/test/test_helpers/test_data_sources/test_toar_data.py index 277a637b7d0418289d2b1f78e5e2731957041bcd..abaec10cc580b592d85d7dcc842616c67777f174 100644 --- a/test/test_helpers/test_data_sources/test_toar_data.py +++ b/test/test_helpers/test_data_sources/test_toar_data.py @@ -1,5 +1,5 @@ from mlair.configuration.join_settings import join_settings -from mlair.helpers.data_sources.toar_data import get_data, create_url +from mlair.helpers.data_sources.toar_data import get_data, create_url, correct_stat_name class TestGetData: @@ -38,3 +38,16 @@ class TestCreateUrl: url = create_url("www.base.edu", "testingservice", param_id=2001, mood="sad", series_id=222) assert url == "www.base.edu/testingservice/2001?mood=sad&series_id=222" + + +class TestCorrectStatName: + + def test_nothing_to_do(self): + assert correct_stat_name("dma8eu") == "dma8eu" + assert correct_stat_name("max") == "max" + + def test_correct_string(self): + assert correct_stat_name("maximum") == "max" + assert correct_stat_name("minimum") == "min" + assert correct_stat_name("average_values") == "mean" +