From 62af3eeeca27b71283eab7d45844791c3735c092 Mon Sep 17 00:00:00 2001
From: leufen1 <l.leufen@fz-juelich.de>
Date: Thu, 15 Jun 2023 18:43:54 +0200
Subject: [PATCH] correct issue that get_data return None instead of raise an
error
---
mlair/helpers/data_sources/data_loader.py | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/mlair/helpers/data_sources/data_loader.py b/mlair/helpers/data_sources/data_loader.py
index 7f61d1b..7f48a30 100644
--- a/mlair/helpers/data_sources/data_loader.py
+++ b/mlair/helpers/data_sources/data_loader.py
@@ -85,7 +85,7 @@ class EmptyQueryResult(Exception):
pass
-def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> Union[Dict, List, str]:
+def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5, timeout_base=60) -> Union[Dict, List, str]:
"""
Download join data using requests framework.
@@ -98,10 +98,10 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) ->
:return: requested data (either as list or dictionary)
"""
url = create_url(**opts)
- for retry in range(max_retries):
+ for retry in range(max_retries + 1):
time.sleep(random.random())
try:
- timeout = 60 * (2 ** retry)
+ timeout = timeout_base * (2 ** retry)
logging.info(f"connect (retry={retry}, timeout={timeout}) {url}")
with TimeTracking(name=url):
session = retries_session(max_retries=0)
@@ -113,8 +113,8 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) ->
except Exception as e:
time.sleep(retry)
logging.debug(f"There was an error for request {url}: {e}")
- if retry + 1 >= max_retries:
- raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}")
+ if retry + 1 >= max_retries:
+ raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}")
def correct_stat_name(stat: str) -> str:
--
GitLab