diff --git a/mlair/helpers/join.py b/mlair/helpers/join.py
index 99689ab110c172c765bdd4bfa203bfd1524e634d..b772a053a7131e5538cbb9da6b03f6dc1c775a4d 100644
--- a/mlair/helpers/join.py
+++ b/mlair/helpers/join.py
@@ -45,7 +45,8 @@ def download_join(station_name: Union[str, List[str]], stat_var: dict, station_t
join_url_base, headers = join_settings(sampling)
# load series information
- vars_dict = load_series_information(station_name, station_type, network_name, join_url_base, headers, data_origin)
+ vars_dict, data_origin = load_series_information(station_name, station_type, network_name, join_url_base, headers,
+ data_origin)
# check if all requested variables are available
if set(stat_var).issubset(vars_dict) is False:
@@ -65,7 +66,7 @@ def download_join(station_name: Union[str, List[str]], stat_var: dict, station_t
for var in _lower_list(sorted(vars_dict.keys())):
if var in stat_var.keys():
- logging.debug('load: {}'.format(var)) # ToDo start here for #206
+ logging.debug('load: {}'.format(var))
# create data link
opts = {'base': join_url_base, 'service': 'stats', 'id': vars_dict[var], 'statistics': stat_var[var],
@@ -133,7 +134,7 @@ def get_data(opts: Dict, headers: Dict) -> Union[Dict, List]:
def load_series_information(station_name: List[str], station_type: str_or_none, network_name: str_or_none,
- join_url_base: str, headers: Dict, data_origin: Dict = None) -> Dict:
+ join_url_base: str, headers: Dict, data_origin: Dict = None) -> [Dict, Dict]:
"""
List all series ids that are available for given station id and network name.
@@ -151,27 +152,30 @@ def load_series_information(station_name: List[str], station_type: str_or_none,
"network_name": network_name, "as_dict": "true",
"columns": "id,network_name,station_id,parameter_name,parameter_label,parameter_attribute"}
station_vars = get_data(opts, headers)
- logging.debug(f"{station_name}: {station_vars}") # ToDo start here for #206
+ logging.debug(f"{station_name}: {station_vars}")
return _select_distinct_series(station_vars, data_origin)
-def _select_distinct_series(vars: List[Dict], data_origin: Dict = None):
+def _select_distinct_series(vars: List[Dict], data_origin: Dict = None) -> [Dict, Dict]:
"""
Select distinct series ids for all variables. Also check if a parameter is from REA or not.
"""
+ data_origin_default = {"cloudcover": "REA", "humidity": "REA", "pblheight": "REA", "press": "REA", "relhum": "REA",
+ "temp": "REA", "totprecip": "REA", "u": "REA", "v": "REA",
+ "no": "", "no2": "", "o3": "", "pm10": "", "so2": ""}
if data_origin is None:
- data_origin = {"cloudcover": "REA", "humidity": "REA", "pblheight": "REA", "press": "REA", "relhum": "REA",
- "temp": "REA", "totprecip": "REA", "u": "REA", "v": "REA",
- "no": "", "no2": "", "o3": "", "pm10": "", "so2": ""}
+ data_origin = {}
# ToDo: maybe press, wdir, wspeed from obs? or also temp, ... ?
selected = {}
for var in vars:
name = var["parameter_name"].lower()
var_attr = var["parameter_attribute"].lower()
+ if name not in data_origin.keys():
+ data_origin.update({name: data_origin_default.get(name, "")})
attr = data_origin.get(name, "").lower()
if var_attr == attr:
selected[name] = var["id"]
- return selected
+ return selected, data_origin
def _save_to_pandas(df: Union[pd.DataFrame, None], data: dict, stat: str, var: str) -> pd.DataFrame:
diff --git a/mlair/run_modules/post_processing.py b/mlair/run_modules/post_processing.py
index 21fbb3edfa2c9665db12f7ea6bb0f9310ca324a1..d125474e2224d4311137702c2796bd89b9f198ee 100644
--- a/mlair/run_modules/post_processing.py
+++ b/mlair/run_modules/post_processing.py
@@ -257,7 +257,7 @@ class PostProcessing(RunEnvironment):
.. note:: Bootstrap plots are only created if bootstraps are evaluated.
"""
- logging.debug("Run plotting routines...")
+ logging.info("Run plotting routines...")
path = self.data_store.get("forecast_path")
plot_list = self.data_store.get("plot_list", "postprocessing")
diff --git a/test/test_join.py b/test/test_join.py
index a9a4c381cbf58a272389b0b11283c8b0cce3ab42..e1df06c2f34ff509e034cd7218433de7403471e7 100644
--- a/test/test_join.py
+++ b/test/test_join.py
@@ -25,7 +25,18 @@ class TestDownloadJoin:
def test_download_empty(self):
with pytest.raises(EmptyQueryResult) as e:
download_join("DEBW107", {"o3": "dma8eu"}, "traffic")
- assert e.value.args[-1] == "No data found in JOIN."
+ assert e.value.args[-1] == "No data found for variables {'o3'} and options station=['DEBW107'], type=traffic," \
+ " network=None, origin={'o3': ''} in JOIN."
+
+ def test_download_incomplete(self):
+ with pytest.raises(EmptyQueryResult) as e:
+ download_join("DEBW107", {"o3": "dma8eu", "o10": "maximum"}, "background")
+ assert e.value.args[-1] == "No data found for variables {'o10'} and options station=['DEBW107'], " \
+ "type=background, network=None, origin={} in JOIN."
+ with pytest.raises(EmptyQueryResult) as e:
+ download_join("DEBW107", {"o3": "dma8eu", "o10": "maximum"}, "background", data_origin={"o10": ""})
+ assert e.value.args[-1] == "No data found for variables {'o10'} and options station=['DEBW107'], " \
+ "type=background, network=None, origin={'o10': ''} in JOIN."
class TestCorrectDataFormat:
@@ -81,15 +92,18 @@ class TestSelectDistinctSeries:
'parameter_label': 'PRESS-REA-MIUB', 'parameter_attribute': 'REA'}]
def test_no_origin_given(self, vars):
- res = _select_distinct_series(vars)
+ res, orig = _select_distinct_series(vars)
assert res == {"no2": 16686, "o3": 16687, "cloudcover": 54036, "temp": 88491, "press": 102660}
+ assert orig == {"no2": "", "o3": "", "cloudcover": "REA", "temp": "REA", "press": "REA"}
def test_different_origins(self, vars):
origin = {"no2": "test", "temp": "", "cloudcover": "REA"}
- res = _select_distinct_series(vars, data_origin=origin)
- assert res == {"o3": 16687, "press": 16692, "temp": 16693, "cloudcover": 54036}
- res = _select_distinct_series(vars, data_origin={})
- assert res == {"no2": 16686, "o3": 16687, "press": 16692, "temp": 16693}
+ res, orig = _select_distinct_series(vars, data_origin=origin)
+ assert res == {"o3": 16687, "press": 102660, "temp": 16693, "cloudcover": 54036}
+ assert orig == {"no2": "test", "o3": "", "cloudcover": "REA", "temp": "", "press": "REA"}
+ res, orig = _select_distinct_series(vars, data_origin={})
+ assert res == {"cloudcover": 54036, "no2": 16686, "o3": 16687, "press": 102660, "temp": 88491}
+ assert orig == {"no2": "", "o3": "", "temp": "REA", "press": "REA", "cloudcover": "REA"}
class TestSaveToPandas: