Merge branch 'lukas_issue296_tech_limit-join-retries' into 'lukas_issue289_feat_cnn-model-class'

Resolve "limit retries in join module"

See merge request !280

mlair/helpers/join.py

@@ -8,6 +8,8 @@ from typing import Iterator, Union, List, Dict
 
 import pandas as pd
 import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 
 from mlair import helpers
 from mlair.configuration.join_settings import join_settings
@@ -129,13 +131,24 @@ def get_data(opts: Dict, headers: Dict) -> Union[Dict, List]:
     :return: requested data (either as list or dictionary)
     """
     url = create_url(**opts)
-    response = requests.get(url, headers=headers)
+    response = retries_session().get(url, headers=headers)
     if response.status_code == 200:
         return response.json()
     else:
         raise EmptyQueryResult(f"There was an error (STATUS {response.status_code}) for request {url}")
 
 
+def retries_session(max_retries=5):
+    retry_strategy = Retry(total=max_retries,
+                           status_forcelist=[429, 500, 502, 503, 504],
+                           method_whitelist=["HEAD", "GET", "OPTIONS"])
+    adapter = HTTPAdapter(max_retries=retry_strategy)
+    http = requests.Session()
+    http.mount("https://", adapter)
+    http.mount("http://", adapter)
+    return http
+
+
 def load_series_information(station_name: List[str], station_type: str_or_none, network_name: str_or_none,
                             join_url_base: str, headers: Dict, data_origin: Dict = None) -> [Dict, Dict]:
     """