Resolve "advanced retry strategy"

Closes #453 (closed)

mlair/helpers/data_sources/data_loader.py

@@ -85,7 +85,7 @@ class EmptyQueryResult(Exception):
     pass
 
 
-def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> Union[Dict, List, str]:
+def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5, timeout_base=60) -> Union[Dict, List, str]:
     """
     Download join data using requests framework.
 
@@ -98,10 +98,11 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) ->
     :return: requested data (either as list or dictionary)
     """
     url = create_url(**opts)
-    for retry in range(max_retries):
+    response_error = None
+    for retry in range(max_retries + 1):
         time.sleep(random.random())
         try:
-            timeout = 60 * (2 ** retry)
+            timeout = timeout_base * (2 ** retry)
             logging.info(f"connect (retry={retry}, timeout={timeout}) {url}")
             with TimeTracking(name=url):
                 session = retries_session(max_retries=0)
@@ -110,11 +111,13 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) ->
                     return response.json() if as_json is True else response.text
                 else:
                     logging.debug(f"There was an error (STATUS {response.status_code}) for request {url}")
+                    response_error = f"STATUS {response.status_code}"
         except Exception as e:
             time.sleep(retry)
             logging.debug(f"There was an error for request {url}: {e}")
-            if retry + 1 >= max_retries:
-                raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}")
+            response_error = e
+        if retry + 1 >= max_retries:
+            raise EmptyQueryResult(f"There was an RetryError for request {url}: {response_error}")
 
 
 def correct_stat_name(stat: str) -> str: