diff --git a/mlair/helpers/join.py b/mlair/helpers/join.py index 8a8ca0b8c964268aa6043312cd1cc88bc0d50544..e0b28660f8d19e260dab5a68deced3af49b05f8e 100644 --- a/mlair/helpers/join.py +++ b/mlair/helpers/join.py @@ -8,6 +8,8 @@ from typing import Iterator, Union, List, Dict import pandas as pd import requests +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry from mlair import helpers from mlair.configuration.join_settings import join_settings @@ -129,13 +131,24 @@ def get_data(opts: Dict, headers: Dict) -> Union[Dict, List]: :return: requested data (either as list or dictionary) """ url = create_url(**opts) - response = requests.get(url, headers=headers) + response = retries_session().get(url, headers=headers) if response.status_code == 200: return response.json() else: raise EmptyQueryResult(f"There was an error (STATUS {response.status_code}) for request {url}") +def retries_session(max_retries=5): + retry_strategy = Retry(total=max_retries, + status_forcelist=[429, 500, 502, 503, 504], + method_whitelist=["HEAD", "GET", "OPTIONS"]) + adapter = HTTPAdapter(max_retries=retry_strategy) + http = requests.Session() + http.mount("https://", adapter) + http.mount("http://", adapter) + return http + + def load_series_information(station_name: List[str], station_type: str_or_none, network_name: str_or_none, join_url_base: str, headers: Dict, data_origin: Dict = None) -> [Dict, Dict]: """