From 1ea8b24f2b5af03d7d08a5c6f92738cd8f69135c Mon Sep 17 00:00:00 2001
From: leufen1 <l.leufen@fz-juelich.de>
Date: Mon, 29 Mar 2021 12:28:15 +0200
Subject: [PATCH] join module now uses a retry strategy, /close #296 on test
success
---
mlair/helpers/join.py | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/mlair/helpers/join.py b/mlair/helpers/join.py
index 8a8ca0b8..e0b28660 100644
--- a/mlair/helpers/join.py
+++ b/mlair/helpers/join.py
@@ -8,6 +8,8 @@ from typing import Iterator, Union, List, Dict
import pandas as pd
import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
from mlair import helpers
from mlair.configuration.join_settings import join_settings
@@ -129,13 +131,24 @@ def get_data(opts: Dict, headers: Dict) -> Union[Dict, List]:
:return: requested data (either as list or dictionary)
"""
url = create_url(**opts)
- response = requests.get(url, headers=headers)
+ response = retries_session().get(url, headers=headers)
if response.status_code == 200:
return response.json()
else:
raise EmptyQueryResult(f"There was an error (STATUS {response.status_code}) for request {url}")
+def retries_session(max_retries=5):
+ retry_strategy = Retry(total=max_retries,
+ status_forcelist=[429, 500, 502, 503, 504],
+ method_whitelist=["HEAD", "GET", "OPTIONS"])
+ adapter = HTTPAdapter(max_retries=retry_strategy)
+ http = requests.Session()
+ http.mount("https://", adapter)
+ http.mount("http://", adapter)
+ return http
+
+
def load_series_information(station_name: List[str], station_type: str_or_none, network_name: str_or_none,
join_url_base: str, headers: Dict, data_origin: Dict = None) -> [Dict, Dict]:
"""
--
GitLab