diff --git a/mlair/helpers/data_sources/data_loader.py b/mlair/helpers/data_sources/data_loader.py index 7f61d1b96818c14494897eb9731124128b7240de..7f48a306a4827cb0b5b9be5c8c0d4dcbbbc886f3 100644 --- a/mlair/helpers/data_sources/data_loader.py +++ b/mlair/helpers/data_sources/data_loader.py @@ -85,7 +85,7 @@ class EmptyQueryResult(Exception): pass -def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> Union[Dict, List, str]: +def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5, timeout_base=60) -> Union[Dict, List, str]: """ Download join data using requests framework. @@ -98,10 +98,10 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> :return: requested data (either as list or dictionary) """ url = create_url(**opts) - for retry in range(max_retries): + for retry in range(max_retries + 1): time.sleep(random.random()) try: - timeout = 60 * (2 ** retry) + timeout = timeout_base * (2 ** retry) logging.info(f"connect (retry={retry}, timeout={timeout}) {url}") with TimeTracking(name=url): session = retries_session(max_retries=0) @@ -113,8 +113,8 @@ def get_data(opts: Dict, headers: Dict, as_json: bool = True, max_retries=5) -> except Exception as e: time.sleep(retry) logging.debug(f"There was an error for request {url}: {e}") - if retry + 1 >= max_retries: - raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}") + if retry + 1 >= max_retries: + raise EmptyQueryResult(f"There was an RetryError for request {url}: {e}") def correct_stat_name(stat: str) -> str: