Skip to content
Snippets Groups Projects
Commit 62af3eee authored by leufen1's avatar leufen1
Browse files

correct issue that get_data return None instead of raise an error

parent 8b41958b
No related branches found
No related tags found
3 merge requests!522filter can now combine obs, forecast, and apriori for first iteration. Further...,!521Resolve "release v2.4.0",!516Resolve "advanced retry strategy"
Pipeline #142645 failed
...@@ -85,7 +85,7 @@ class EmptyQueryResult(Exception): ...@@ -85,7 +85,7 @@ class EmptyQueryResult(Exception):
pass pass
def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> Union[Dict, List, str]: def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5, timeout_base=60) -> Union[Dict, List, str]:
""" """
Download join data using requests framework. Download join data using requests framework.
...@@ -98,10 +98,10 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> ...@@ -98,10 +98,10 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) ->
:return: requested data (either as list or dictionary) :return: requested data (either as list or dictionary)
""" """
url = create_url(**opts) url = create_url(**opts)
for retry in range(max_retries): for retry in range(max_retries + 1):
time.sleep(random.random()) time.sleep(random.random())
try: try:
timeout = 60 * (2 ** retry) timeout = timeout_base * (2 ** retry)
logging.info(f"connect (retry={retry}, timeout={timeout}) {url}") logging.info(f"connect (retry={retry}, timeout={timeout}) {url}")
with TimeTracking(name=url): with TimeTracking(name=url):
session = retries_session(max_retries=0) session = retries_session(max_retries=0)
...@@ -113,8 +113,8 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> ...@@ -113,8 +113,8 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) ->
except Exception as e: except Exception as e:
time.sleep(retry) time.sleep(retry)
logging.debug(f"There was an error for request {url}: {e}") logging.debug(f"There was an error for request {url}: {e}")
if retry + 1 >= max_retries: if retry + 1 >= max_retries:
raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}") raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}")
def correct_stat_name(stat: str) -> str: def correct_stat_name(stat: str) -> str:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment