add: script to test response time of analysis service for different timescales

tests/benchmark.py

+from datetime import datetime, timedelta
+import time
+import requests
+from pathlib import Path
+
+from toargridding.toar_rest_client import AnalysisServiceDownload
+from toargridding.metadata import Metadata, TimeSample, TOARVariable
+
+start = datetime(2016, 3, 1)
+end = datetime(2016, 3, 3)
+
+SAMPLING = "daily"  # FIXME check monthly !!!
+STATISTICS_ENDPOINT = "https://toar-data.fz-juelich.de/api/v2/analysis/statistics/"
+STATISTIC = "mean"
+TEST_ROOT = Path(__file__).parent
+
+
+def get_toar_response(start, end):
+    end_with_padding = end + timedelta(1)
+
+    response = requests.get(
+        STATISTICS_ENDPOINT,
+        params={
+            "daterange": f"{start.isoformat()},{end_with_padding.isoformat()}",  # 1-year
+            "variable_id": 5,
+            "statistics": STATISTIC,
+            "sampling": SAMPLING,
+            "min_data_capture": 0,
+            "limit": "None",  # get all timeseries
+            "format": "by_statistic",
+            "metadata_scheme": "basic",
+        },
+    )
+    return wait_for_data(response)
+
+
+def wait_for_data(response):
+    tries = 0
+    while True:
+        print(f"n tries: {tries}")
+        if response.headers["Content-Type"] == "application/zip":
+            break
+        else:
+            status_endpoint = response.json()["status"]
+            time.sleep(60)
+            try:
+                response = requests.get(status_endpoint)
+            except ConnectionError:
+                print("temporarly no connection")
+        tries += 1
+
+    return tries
+
+
+def time_toar_response(start, end):
+    print(f"starting request for {start}-{end}")
+
+    timer_start = datetime.now()
+    tries = get_toar_response(start, end)
+    timer_end = datetime.now()
+    response_time = timer_end - timer_start
+
+    print(f"response time for {start}-{end}: {response_time} ({tries} tries)")
+
+
+def wait_for_client_response(client, sample):
+    for half_hours in range(20):
+        try:
+            client.get_data(sample)
+        except RuntimeError:
+            print(f"time out after 30min, try {half_hours+1}")
+
+
+def time_rest_client_response(start, end):
+    print(f"starting request for {start}-{end}")
+
+    rest_client = AnalysisServiceDownload(
+        STATISTICS_ENDPOINT,
+        TEST_ROOT / "temp_data_cache",
+        TEST_ROOT / "temp_data",
+        use_downloaded=False,
+    )
+    time_window = TimeSample(start, end, SAMPLING)
+    sample = Metadata.construct("mole_fraction_of_ozone_in_air", STATISTIC, time_window)
+
+    timer_start = datetime.now()
+    wait_for_client_response(rest_client, sample)
+    timer_end = datetime.now()
+    response_time = timer_end - timer_start
+
+    print(f"response time for {start}-{end}: {response_time} (tries)")
+
+
+if __name__ == "__main__":
+    time_windows = [
+        (datetime(2010, 1, 1), datetime(2010, 1, 2)),
+        (datetime(2010, 1, 1), datetime(2010, 1, 8)),
+        (datetime(2010, 1, 1), datetime(2010, 1, 31)),
+        (datetime(2010, 1, 1), datetime(2010, 12, 31)),
+        (datetime(2010, 1, 1), datetime(2015, 12, 31)),
+    ]
+    for start, end in time_windows:
+        print("rest client")
+        time_rest_client_response(start, end)