diff --git a/tests/produce_data_manyStations.ipynb b/tests/produce_data_manyStations.ipynb
index c9734b8f4de65b42cd1280ed052ceb5bb494156f..dbb7235ed3f13bc7837efa53661fbea97c0700a2 100644
--- a/tests/produce_data_manyStations.ipynb
+++ b/tests/produce_data_manyStations.ipynb
@@ -10,7 +10,7 @@
"from collections import namedtuple\n",
"from pathlib import Path\n",
"\n",
- "from toargridding.toar_rest_client import AnalysisServiceDownload\n",
+ "from toargridding.toar_rest_client import AnalysisServiceDownload, Connection\n",
"from toargridding.grids import RegularGrid\n",
"from toargridding.gridding import get_gridded_toar_data\n",
"from toargridding.metadata import TimeSample"
@@ -60,6 +60,8 @@
"result_basepath.mkdir(exist_ok=True)\n",
"analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath, use_downloaded=True)\n",
"\n",
+ "Connection.DEBUG=True\n",
+ "\n",
"for person, config in configs.items():\n",
" datasets, metadatas = get_gridded_toar_data(\n",
" analysis_service=analysis_service,\n",
diff --git a/tests/produce_data_withOptional.ipynb b/tests/produce_data_withOptional.ipynb
index 64d8954459aca498daae857caf60c4f59dbe5003..228a577e5a9cbfd6362c8583b6f2d8a2d8d32f2a 100644
--- a/tests/produce_data_withOptional.ipynb
+++ b/tests/produce_data_withOptional.ipynb
@@ -10,7 +10,7 @@
"from collections import namedtuple\n",
"from pathlib import Path\n",
"\n",
- "from toargridding.toar_rest_client import AnalysisServiceDownload\n",
+ "from toargridding.toar_rest_client import AnalysisServiceDownload, Connection\n",
"from toargridding.grids import RegularGrid\n",
"from toargridding.gridding import get_gridded_toar_data\n",
"from toargridding.metadata import TimeSample"
@@ -76,6 +76,8 @@
"result_basepath.mkdir(exist_ok=True)\n",
"analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath, use_downloaded=True)\n",
"\n",
+ "Connection.DEBUG=True\n",
+ "\n",
"for person, config in configs.items():\n",
" datasets, metadatas = get_gridded_toar_data(\n",
" analysis_service=analysis_service,\n",
diff --git a/toargridding/toar_rest_client.py b/toargridding/toar_rest_client.py
index 4839c9c137a0a528c84d2a44272533bea92862ad..0854bc39d0de6aa185d0fbca9aba159651e22de1 100644
--- a/toargridding/toar_rest_client.py
+++ b/toargridding/toar_rest_client.py
@@ -153,6 +153,7 @@ class Cache:
class Connection:
+ DEBUG = False
def __init__(self, endpoint, cache_dir):
"""connection to the rest API of the TOAR database
@@ -175,7 +176,7 @@ class Connection:
# max wait time is 30min
self.wait_seconds = [minutes * 60 for minutes in (5, 5, 5, 5, 5, 5)]
- def get(self, query_options : QueryOptions):
+ def get(self, query_options : QueryOptions) -> requests.models.Response:
"""get results for a request.
This is the main function to obtained data from the TOAR DB. It will start requests or lookup if an already started requests is finished.
@@ -193,9 +194,7 @@ class Connection:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
print(f"\tconnection error ({e.response.status_code}: {e.response.reason}). Trying again later")
- self.printExecption(e)
- print(response.content)
- print(response.json())
+ self.printExecption(e, response)
continue
#are our results ready to obtain?
if response.headers["Content-Type"] == "application/zip":
@@ -206,7 +205,7 @@ class Connection:
f"No data available after {sum(self.wait_seconds) / 60} minutes. retry later."
)
- def get_status_endpoint(self, query_options: QueryOptions):
+ def get_status_endpoint(self, query_options: QueryOptions) -> str:
"""get endpoint to results of a request
This function checks if the request is already known and has been submitted to the TOAD DB.
@@ -231,10 +230,8 @@ class Connection:
raise RuntimeError("Connection to TAORDB timed out (ReadTimeout) while checking cached status point. Please try again later.")
except requests.exceptions.HTTPError as e:
#TODO add detailed processing: What was the reason for the error? Do we really need to create a new request or is there another problem, that might resolve by simply waiting
- print(f"A connection error occurred:")
- self.printExecption(e)
- print(response.content)
- print(response.json())
+ print(f"A HTTP error occurred:")
+ self.printExecption(e, response)
print(f"Status Endpoint: {status_endpoint}")
#use inverse order for saving. the status endpoint should be more unique
self.cache_backup.put(status_endpoint, query_options.cache_key)
@@ -251,7 +248,7 @@ class Connection:
status_endpoint = self.query_for_status_endpoint(query_options)
return status_endpoint
- def query_for_status_endpoint(self, query_options: QueryOptions):
+ def query_for_status_endpoint(self, query_options: QueryOptions) -> str:
"""create and new request to the TOAR DB.
Adds the status endpoint of the request to the cache.
@@ -267,10 +264,7 @@ class Connection:
response = self.wait_and_get(self.endpoint, asdict(query_options, dict_factory=quarryToDict))
except requests.exceptions.HTTPError as e:
print(f"A connection error occurred:")
- self.printExecption(e)
- print(f"{response=}")
- print(f"{response.content=}")
- print(response.json())
+ self.printExecption(e, response)
raise e
except requests.exceptions.ReadTimeout as e:
print("Caught read timeout.")
@@ -288,7 +282,7 @@ class Connection:
def wait_and_get(
self, endpoint : str, query_options : Dict =None, wait_secs=None, timeout=(3.05, 20)
- ):
+ ) -> requests.models.Response:
"""accesses given endpoint
Parameters:
@@ -306,10 +300,14 @@ class Connection:
time.sleep(wait_secs)
return requests.get(endpoint, params=query_options, timeout=timeout)
- def printExecption(self, e : requests.exceptions.HTTPError):
- print(f"Status Code: {e.response.status_code}")
- print(f"Reason: {e.response.reason}")
- print(f"Text: {e.response.text}")
+ def printExecption(self, e : requests.exceptions.HTTPError, response : requests.Response):
+ if Connection.DEBUG:
+ print(f"Status Code: {e.response.status_code}")
+ print(f"Reason: {e.response.reason}")
+ print(f"Text: {e.response.text}")
+ print(f"{response=}")
+ print(f"{response.content=}")
+ print(response.json())
class AnalysisService: