Skip to content
Snippets Groups Projects
Commit 5e9aa8fe authored by leufen1's avatar leufen1
Browse files

Merge branch 'lukas_issue453_refac_advanced-retry-strategy' into...

Merge branch 'lukas_issue453_refac_advanced-retry-strategy' into michael_issue450_feat_load-ifs-data
parents 1863bc19 8b41958b
No related branches found
No related tags found
3 merge requests!522filter can now combine obs, forecast, and apriori for first iteration. Further...,!521Resolve "release v2.4.0",!517Resolve "load ifs data"
......@@ -3,6 +3,8 @@ __date__ = '2023-06-01'
import logging
from typing import Dict, Union, List
import time
import random
import requests
from requests.adapters import HTTPAdapter, Retry
......@@ -93,7 +95,7 @@ class EmptyQueryResult(Exception):
pass
def get_data(opts: Dict, headers: Dict, as_json: bool = True) -> Union[Dict, List, str]:
def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> Union[Dict, List, str]:
"""
Download join data using requests framework.
......@@ -106,15 +108,23 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True) -> Union[Dict, Lis
:return: requested data (either as list or dictionary)
"""
url = create_url(**opts)
try:
with TimeTracking(name=url):
response = retries_session().get(url, headers=headers, timeout=(5, None)) # timeout=(open, read)
if response.status_code == 200:
return response.json() if as_json is True else response.text
else:
raise EmptyQueryResult(f"There was an error (STATUS {response.status_code}) for request {url}")
except requests.exceptions.RetryError as e:
raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}")
for retry in range(max_retries):
time.sleep(random.random())
try:
timeout = 60 * (2 ** retry)
logging.info(f"connect (retry={retry}, timeout={timeout}) {url}")
with TimeTracking(name=url):
session = retries_session(max_retries=0)
response = session.get(url, headers=headers, timeout=(5, timeout)) # timeout=(open, read)
if response.status_code == 200:
return response.json() if as_json is True else response.text
else:
logging.debug(f"There was an error (STATUS {response.status_code}) for request {url}")
except Exception as e:
time.sleep(retry)
logging.debug(f"There was an error for request {url}: {e}")
if retry + 1 >= max_retries:
raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}")
def correct_stat_name(stat: str) -> str:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment