From dcdccc09c1ada035d0b29d79396c52c0683a83df Mon Sep 17 00:00:00 2001
From: leufen1 <l.leufen@fz-juelich.de>
Date: Tue, 12 Jul 2022 12:36:40 +0200
Subject: [PATCH] only append timeseries data if len > 0
---
mlair/helpers/data_sources/join.py | 16 +---------------
mlair/helpers/data_sources/toar_data.py | 14 ++++++++++++++
mlair/helpers/data_sources/toar_data_v2.py | 8 ++++----
test/test_helpers/test_data_sources/test_join.py | 14 +-------------
.../test_data_sources/test_toar_data.py | 15 ++++++++++++++-
5 files changed, 34 insertions(+), 33 deletions(-)
diff --git a/mlair/helpers/data_sources/join.py b/mlair/helpers/data_sources/join.py
index df3b358e..a978b271 100644
--- a/mlair/helpers/data_sources/join.py
+++ b/mlair/helpers/data_sources/join.py
@@ -80,7 +80,7 @@ def download_join(station_name: Union[str, List[str]], stat_var: dict, station_t
data = correct_data_format(data)
# correct namespace of statistics
- stat = _correct_stat_name(stat_var[var])
+ stat = toar_data.correct_stat_name(stat_var[var])
# store data in pandas dataframe
df = _save_to_pandas(df, data, stat, var)
@@ -344,20 +344,6 @@ def _save_to_pandas(df: Union[pd.DataFrame, None], data: dict, stat: str, var: s
return df
-def _correct_stat_name(stat: str) -> str:
- """
- Map given statistic name to new namespace defined by mapping dict.
-
- Return given name stat if not element of mapping namespace.
-
- :param stat: namespace from JOIN server
-
- :return: stat mapped to local namespace
- """
- mapping = {'average_values': 'mean', 'maximum': 'max', 'minimum': 'min'}
- return mapping.get(stat, stat)
-
-
def _lower_list(args: List[str]) -> Iterator[str]:
"""
Lower all elements of given list.
diff --git a/mlair/helpers/data_sources/toar_data.py b/mlair/helpers/data_sources/toar_data.py
index c2aecfd3..27522855 100644
--- a/mlair/helpers/data_sources/toar_data.py
+++ b/mlair/helpers/data_sources/toar_data.py
@@ -112,3 +112,17 @@ def merge_toar_join(df_toar, df_join, sampling):
full_data = df_toar.reindex(full_time)
full_data.update(df_join, overwrite=False)
return full_data
+
+
+def correct_stat_name(stat: str) -> str:
+ """
+ Map given statistic name to new namespace defined by mapping dict.
+
+ Return given name stat if not element of mapping namespace.
+
+ :param stat: namespace from JOIN server
+
+ :return: stat mapped to local namespace
+ """
+ mapping = {'average_values': 'mean', 'maximum': 'max', 'minimum': 'min'}
+ return mapping.get(stat, stat)
diff --git a/mlair/helpers/data_sources/toar_data_v2.py b/mlair/helpers/data_sources/toar_data_v2.py
index a592d2d1..0fa53a7e 100644
--- a/mlair/helpers/data_sources/toar_data_v2.py
+++ b/mlair/helpers/data_sources/toar_data_v2.py
@@ -13,8 +13,7 @@ from timezonefinder import TimezoneFinder
from mlair.configuration.toar_data_v2_settings import toar_data_v2_settings
from mlair.helpers import to_list
-from mlair.helpers.data_sources.toar_data import EmptyQueryResult, get_data
-from mlair.helpers.data_sources import join
+from mlair.helpers.data_sources.toar_data import EmptyQueryResult, get_data, correct_stat_name
str_or_none = Union[str, None]
@@ -174,8 +173,9 @@ def load_timeseries_data(timeseries_meta, url_base, opts, headers, sampling):
res = get_data(opts, headers, as_json=False)
data = pd.read_csv(StringIO(res), comment="#", index_col="datetime", parse_dates=True,
infer_datetime_format=True)
- data = data[join._correct_stat_name(opts.get("names", "value"))].rename(meta["variable"]["name"])
- coll.append(data)
+ if len(data.index) > 0:
+ data = data[correct_stat_name(opts.get("names", "value"))].rename(meta["variable"]["name"])
+ coll.append(data)
return coll
diff --git a/test/test_helpers/test_data_sources/test_join.py b/test/test_helpers/test_data_sources/test_join.py
index 0a9715f5..f9b12f5a 100644
--- a/test/test_helpers/test_data_sources/test_join.py
+++ b/test/test_helpers/test_data_sources/test_join.py
@@ -3,7 +3,7 @@ from typing import Iterable
import pytest
from mlair.helpers.data_sources.join import *
-from mlair.helpers.data_sources.join import _save_to_pandas, _correct_stat_name, _lower_list, _select_distinct_series, \
+from mlair.helpers.data_sources.join import _save_to_pandas, _lower_list, _select_distinct_series, \
_select_distinct_data_origin, _select_distinct_network
from mlair.configuration.join_settings import join_settings
from mlair.helpers.testing import check_nested_equality
@@ -334,18 +334,6 @@ class TestSaveToPandas:
assert pd.testing.assert_frame_equal(create_df, _save_to_pandas(None, data, 'mean', 'cloudcover')) is None
-class TestCorrectStatName:
-
- def test_nothing_to_do(self):
- assert _correct_stat_name("dma8eu") == "dma8eu"
- assert _correct_stat_name("max") == "max"
-
- def test_correct_string(self):
- assert _correct_stat_name("maximum") == "max"
- assert _correct_stat_name("minimum") == "min"
- assert _correct_stat_name("average_values") == "mean"
-
-
class TestLowerList:
def test_string_lowering(self):
diff --git a/test/test_helpers/test_data_sources/test_toar_data.py b/test/test_helpers/test_data_sources/test_toar_data.py
index 277a637b..abaec10c 100644
--- a/test/test_helpers/test_data_sources/test_toar_data.py
+++ b/test/test_helpers/test_data_sources/test_toar_data.py
@@ -1,5 +1,5 @@
from mlair.configuration.join_settings import join_settings
-from mlair.helpers.data_sources.toar_data import get_data, create_url
+from mlair.helpers.data_sources.toar_data import get_data, create_url, correct_stat_name
class TestGetData:
@@ -38,3 +38,16 @@ class TestCreateUrl:
url = create_url("www.base.edu", "testingservice", param_id=2001, mood="sad", series_id=222)
assert url == "www.base.edu/testingservice/2001?mood=sad&series_id=222"
+
+
+class TestCorrectStatName:
+
+ def test_nothing_to_do(self):
+ assert correct_stat_name("dma8eu") == "dma8eu"
+ assert correct_stat_name("max") == "max"
+
+ def test_correct_string(self):
+ assert correct_stat_name("maximum") == "max"
+ assert correct_stat_name("minimum") == "min"
+ assert correct_stat_name("average_values") == "mean"
+
--
GitLab