Resolve "Use Toar statistics api v2"

Closes #454 (closed)

mlair/helpers/data_sources/data_loader.py

@@ -85,6 +85,36 @@ class EmptyQueryResult(Exception):
     pass
 
 
+def get_data_with_query(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5, timeout_base=60) -> bytes:
+    """
+    Download data from statistics rest api. This API is based on three steps: (1) post query and retrieve job id, (2)
+    read status of id until finished, (3) download data with job id.
+    """
+    url = create_url(**opts)
+    response_error = None
+    for retry in range(max_retries + 1):
+        time.sleep(random.random())
+        try:
+            timeout = timeout_base * (2 ** retry)
+            logging.info(f"connect (retry={retry}, timeout={timeout}) {url}")
+            start_time = time.time()
+            with TimeTracking(name=url):
+                session = retries_session(max_retries=0)
+                response = session.get(url, headers=headers, timeout=(5, 5))  # timeout=(open, read)
+                while (time.time() - start_time) < timeout:
+                    response = requests.get(response.json()["status"], timeout=(5, 5))
+                    if response.history:
+                        break
+                    time.sleep(2)
+                return response.content
+        except Exception as e:
+            time.sleep(retry)
+            logging.debug(f"There was an error for request {url}: {e}")
+            response_error = e
+        if retry + 1 >= max_retries:
+            raise EmptyQueryResult(f"There was an RetryError for request {url}: {response_error}")
+
+
 def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5, timeout_base=60) -> Union[Dict, List, str]:
     """
     Download join data using requests framework.