from datetime import datetime, timedelta import time import requests from pathlib import Path from toargridding.toar_rest_client import AnalysisServiceDownload from toargridding.metadata import Metadata, TimeSample, TOARVariable start = datetime(2016, 3, 1) end = datetime(2016, 3, 3) SAMPLING = "daily" # FIXME check monthly !!! STATISTICS_ENDPOINT = "https://toar-data.fz-juelich.de/api/v2/analysis/statistics/" STATISTIC = "mean" TEST_ROOT = Path(__file__).parent def get_toar_response(start, end): end_with_padding = end + timedelta(1) response = requests.get( STATISTICS_ENDPOINT, params={ "daterange": f"{start.isoformat()},{end_with_padding.isoformat()}", # 1-year "variable_id": 5, "statistics": STATISTIC, "sampling": SAMPLING, "min_data_capture": 0, "limit": "None", # get all timeseries "format": "by_statistic", "metadata_scheme": "basic", }, ) return wait_for_data(response) def wait_for_data(response): tries = 0 while True: print(f"n tries: {tries}") if response.headers["Content-Type"] == "application/zip": break else: status_endpoint = response.json()["status"] time.sleep(60) try: response = requests.get(status_endpoint) except ConnectionError: print("temporarly no connection") tries += 1 return tries def time_toar_response(start, end): print(f"starting request for {start}-{end}") timer_start = datetime.now() tries = get_toar_response(start, end) timer_end = datetime.now() response_time = timer_end - timer_start print(f"response time for {start}-{end}: {response_time} ({tries} tries)") def wait_for_client_response(client, sample): for half_hours in range(20): try: client.get_data(sample) except RuntimeError: print(f"time out after 30min, try {half_hours+1}") def time_rest_client_response(start, end): print(f"starting request for {start}-{end}") rest_client = AnalysisServiceDownload( STATISTICS_ENDPOINT, TEST_ROOT / "temp_data_cache", TEST_ROOT / "temp_data", use_downloaded=False, ) time_window = TimeSample(start, end, SAMPLING) sample = Metadata.construct("mole_fraction_of_ozone_in_air", STATISTIC, time_window) timer_start = datetime.now() wait_for_client_response(rest_client, sample) timer_end = datetime.now() response_time = timer_end - timer_start print(f"response time for {start}-{end}: {response_time} (tries)") if __name__ == "__main__": time_windows = [ (datetime(2010, 1, 1), datetime(2010, 1, 2)), (datetime(2010, 1, 1), datetime(2010, 1, 8)), (datetime(2010, 1, 1), datetime(2010, 1, 31)), (datetime(2010, 1, 1), datetime(2010, 12, 31)), (datetime(2010, 1, 1), datetime(2015, 12, 31)), ] for start, end in time_windows: print("rest client") time_rest_client_response(start, end)