{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2024-05-07 14:38:11.226359\n"
]
}
],
"source": [
"from datetime import datetime, timedelta\n",
"\n",
"from toargridding.metadata import TimeSample, Metadata\n",
"\n",
"sampling = \"daily\" # FIXME check monthly !!!\n",
"start = datetime(2010, 1, 1)\n",
"end = datetime(2011, 1, 1)\n",
"\n",
"statistics_endpoint = \"https://toar-data.fz-juelich.de/api/v2/analysis/statistics/\"\n",
"statistic = \"mean\"\n",
"\n",
"time = TimeSample(start, end, sampling=sampling)\n",
"# { \"station_type_of_area\" : \"urban\" } category is not known\n",
"metadata = Metadata.construct(\"mole_fraction_of_ozone_in_air\", time, statistic, { \"toar1_category\" : \"Urban\"})#\n",
"\n",
"start_time = datetime.now()\n",
"print(start_time)"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"[('daterange', '2010-01-01T00:00:00,2011-01-02T00:00:00'), ('variable_id', '5'), ('statistics', 'mean'), ('sampling', 'daily'), ('min_data_capture', '0'), ('metadata_scheme', 'basic'), ('limit', 'None'), ('format', 'by_statistic'), ('moreOptions', {'station_type_of_area': 'urban'})]\n",
"query not in cache\n",
"[('daterange', '2010-01-01T00:00:00,2011-01-02T00:00:00'), ('variable_id', '5'), ('statistics', 'mean'), ('sampling', 'daily'), ('min_data_capture', '0'), ('metadata_scheme', 'basic'), ('limit', 'None'), ('format', 'by_statistic'), ('moreOptions', {'station_type_of_area': 'urban'})]\n"
]
},
{
"ename": "RuntimeError",
"evalue": "Request was not successful. Response by TOAR database: {\"detail\":\"'An unknown argument was received: station_type_of_area.'\"}",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:236\u001b[0m, in \u001b[0;36mConnection.query_for_status_endpoint\u001b[0;34m(self, query_options)\u001b[0m\n\u001b[1;32m 235\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 236\u001b[0m status_endpoint \u001b[38;5;241m=\u001b[39m \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstatus\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m 237\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n",
"\u001b[0;31mKeyError\u001b[0m: 'status'",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mRuntimeError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[2], line 14\u001b[0m\n\u001b[1;32m 10\u001b[0m download_dir\u001b[38;5;241m.\u001b[39mmkdir(parents\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 12\u001b[0m analysis_service \u001b[38;5;241m=\u001b[39m AnalysisServiceDownload(statistics_endpoint, cache_dir, download_dir)\n\u001b[0;32m---> 14\u001b[0m results \u001b[38;5;241m=\u001b[39m \u001b[43manalysis_service\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmetadata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 16\u001b[0m end_time \u001b[38;5;241m=\u001b[39m datetime\u001b[38;5;241m.\u001b[39mnow()\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28mprint\u001b[39m(end_time\u001b[38;5;241m-\u001b[39mstart_time)\n",
"File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:294\u001b[0m, in \u001b[0;36mAnalysisService.get_data\u001b[0;34m(self, metadata)\u001b[0m\n\u001b[1;32m 280\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_data\u001b[39m(\u001b[38;5;28mself\u001b[39m, metadata: Metadata) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m AnalysisRequestResult:\n\u001b[1;32m 281\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"main function to obtain data from the TOAR DB\u001b[39;00m\n\u001b[1;32m 282\u001b[0m \n\u001b[1;32m 283\u001b[0m \u001b[38;5;124;03m Handles requesting and loading of data into memory as soon as they are available.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 291\u001b[0m \u001b[38;5;124;03m Requested data and statistics, station coordinates and metadata of the request\u001b[39;00m\n\u001b[1;32m 292\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 294\u001b[0m timeseries, timeseries_metadata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_timeseries_and_metadata\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmetadata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 295\u001b[0m coords \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_clean_coords(timeseries_metadata)\n\u001b[1;32m 296\u001b[0m timeseries \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_clean_timeseries(timeseries, metadata)\n",
"File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:420\u001b[0m, in \u001b[0;36mAnalysisServiceDownload.get_timeseries_and_metadata\u001b[0;34m(self, metadata)\u001b[0m\n\u001b[1;32m 417\u001b[0m needs_fresh_download \u001b[38;5;241m=\u001b[39m (\u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muse_downloaded) \u001b[38;5;129;01mor\u001b[39;00m (\u001b[38;5;129;01mnot\u001b[39;00m filename\u001b[38;5;241m.\u001b[39mis_file())\n\u001b[1;32m 419\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m needs_fresh_download:\n\u001b[0;32m--> 420\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery_options\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 421\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(filename, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mw+b\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m downloaded_file:\n\u001b[1;32m 422\u001b[0m downloaded_file\u001b[38;5;241m.\u001b[39mwrite(response\u001b[38;5;241m.\u001b[39mcontent)\n",
"File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:184\u001b[0m, in \u001b[0;36mConnection.get\u001b[0;34m(self, query_options)\u001b[0m\n\u001b[1;32m 177\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget\u001b[39m(\u001b[38;5;28mself\u001b[39m, query_options : QueryOptions):\n\u001b[1;32m 178\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"get results for a request.\u001b[39;00m\n\u001b[1;32m 179\u001b[0m \n\u001b[1;32m 180\u001b[0m \u001b[38;5;124;03m This is the main function to obtained data from the TOAR DB. It will start requests or lookup if an already started requests is finished.\u001b[39;00m\n\u001b[1;32m 181\u001b[0m \n\u001b[1;32m 182\u001b[0m \u001b[38;5;124;03m Throws an exception, if the results are not available after the waiting time. A restart of the function continues the regular lookup for results.\u001b[39;00m\n\u001b[1;32m 183\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 184\u001b[0m status_endpoint \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_status_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery_options\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 186\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, wait_time \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwait_seconds):\n\u001b[1;32m 187\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtry: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, wait_time: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mwait_time\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
"File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:221\u001b[0m, in \u001b[0;36mConnection.get_status_endpoint\u001b[0;34m(self, query_options)\u001b[0m\n\u001b[1;32m 218\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 219\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mquery not in cache\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 221\u001b[0m status_endpoint \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery_for_status_endpoint\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery_options\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 222\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m status_endpoint\n",
"File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:238\u001b[0m, in \u001b[0;36mConnection.query_for_status_endpoint\u001b[0;34m(self, query_options)\u001b[0m\n\u001b[1;32m 236\u001b[0m status_endpoint \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mjson()[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mstatus\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m 237\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m:\n\u001b[0;32m--> 238\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRequest was not successful. Response by TOAR database: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mresponse\u001b[38;5;241m.\u001b[39mtext\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 239\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcache\u001b[38;5;241m.\u001b[39mput(query_options\u001b[38;5;241m.\u001b[39mcache_key, status_endpoint)\n\u001b[1;32m 241\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m status_endpoint\n",
"\u001b[0;31mRuntimeError\u001b[0m: Request was not successful. Response by TOAR database: {\"detail\":\"'An unknown argument was received: station_type_of_area.'\"}"
]
}
],
"source": [
"from pathlib import Path\n",
"from toargridding.toar_rest_client import AnalysisServiceDownload\n",
"\n",
"\n",
"#creation of output directories\n",
"toargridding_base_path = Path(\".\")\n",
"cache_dir = toargridding_base_path / \"results\"\n",
"download_dir = toargridding_base_path / \"data\"\n",
"cache_dir.mkdir(parents=True, exist_ok=True)\n",
"download_dir.mkdir(parents=True, exist_ok=True)\n",
"\n",
"analysis_service = AnalysisServiceDownload(statistics_endpoint, cache_dir, download_dir)\n",
"\n",
"results = analysis_service.get_data(metadata)\n",
"\n",
"end_time = datetime.now()\n",
"print(end_time-start_time)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "toargridding-g-KQ1Hyq-py3.10",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}