diff --git a/tests/produce_data.ipynb b/tests/produce_data.ipynb index 4e0d84565fa1b84ff0243f40620896abc3441572..2490469ac379b360e02aacae6099a419f36a03b3 100644 --- a/tests/produce_data.ipynb +++ b/tests/produce_data.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 22, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -18,20 +18,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<toargridding.grids.RegularGrid at 0x7f024eb0a5d0>" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "#creation of request.\n", "\n", @@ -67,36 +56,9 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Performing request to TOAR DB\n", - "load status endpoint from cache\n", - "try: 1, wait_time: 300\n", - "try: 2, wait_time: 300\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[24], line 14\u001b[0m\n\u001b[1;32m 11\u001b[0m analysis_service \u001b[38;5;241m=\u001b[39m AnalysisServiceDownload(stats_endpoint\u001b[38;5;241m=\u001b[39mstats_endpoint, cache_dir\u001b[38;5;241m=\u001b[39mcache_basepath, sample_dir\u001b[38;5;241m=\u001b[39mresult_basepath, use_downloaded\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m 13\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m person, config \u001b[38;5;129;01min\u001b[39;00m configs\u001b[38;5;241m.\u001b[39mitems():\n\u001b[0;32m---> 14\u001b[0m datasets, metadatas \u001b[38;5;241m=\u001b[39m \u001b[43mget_gridded_toar_data\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 15\u001b[0m \u001b[43m \u001b[49m\u001b[43manalysis_service\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43manalysis_service\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 16\u001b[0m \u001b[43m \u001b[49m\u001b[43mgrid\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgrid\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 17\u001b[0m \u001b[43m \u001b[49m\u001b[43mtime\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtime\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 18\u001b[0m \u001b[43m \u001b[49m\u001b[43mvariables\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvariables\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 19\u001b[0m \u001b[43m \u001b[49m\u001b[43mstats\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstats\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 20\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mconfig\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmoreOptions\u001b[49m\n\u001b[1;32m 21\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 23\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m dataset, metadata \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(datasets, metadatas):\n\u001b[1;32m 24\u001b[0m dataset\u001b[38;5;241m.\u001b[39mto_netcdf(result_basepath \u001b[38;5;241m/\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmetadata\u001b[38;5;241m.\u001b[39mget_id()\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.nc\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/gridding.py:49\u001b[0m, in \u001b[0;36mget_gridded_toar_data\u001b[0;34m(analysis_service, grid, time, variables, stats, **kwargs)\u001b[0m\n\u001b[1;32m 47\u001b[0m datasets \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 48\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m metadata \u001b[38;5;129;01min\u001b[39;00m metadatas: \u001b[38;5;66;03m# standart_name ?\u001b[39;00m\n\u001b[0;32m---> 49\u001b[0m data \u001b[38;5;241m=\u001b[39m \u001b[43manalysis_service\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmetadata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 50\u001b[0m ds \u001b[38;5;241m=\u001b[39m grid\u001b[38;5;241m.\u001b[39mas_xarray(data)\n\u001b[1;32m 51\u001b[0m datasets\u001b[38;5;241m.\u001b[39mappend(ds)\n", - "File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:307\u001b[0m, in \u001b[0;36mAnalysisService.get_data\u001b[0;34m(self, metadata)\u001b[0m\n\u001b[1;32m 293\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_data\u001b[39m(\u001b[38;5;28mself\u001b[39m, metadata: Metadata) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m AnalysisRequestResult:\n\u001b[1;32m 294\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"main function to obtain data from the TOAR DB\u001b[39;00m\n\u001b[1;32m 295\u001b[0m \n\u001b[1;32m 296\u001b[0m \u001b[38;5;124;03m Handles requesting and loading of data into memory as soon as they are available.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 304\u001b[0m \u001b[38;5;124;03m Requested data and statistics, station coordinates and metadata of the request\u001b[39;00m\n\u001b[1;32m 305\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 307\u001b[0m timeseries, timeseries_metadata \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_timeseries_and_metadata\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmetadata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 308\u001b[0m coords \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_clean_coords(timeseries_metadata)\n\u001b[1;32m 309\u001b[0m timeseries \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_clean_timeseries(timeseries, metadata)\n", - "File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:447\u001b[0m, in \u001b[0;36mAnalysisServiceDownload.get_timeseries_and_metadata\u001b[0;34m(self, metadata)\u001b[0m\n\u001b[1;32m 445\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m needs_fresh_download:\n\u001b[1;32m 446\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPerforming request to TOAR DB\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 447\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery_options\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 448\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(filename, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mw+b\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m downloaded_file:\n\u001b[1;32m 449\u001b[0m downloaded_file\u001b[38;5;241m.\u001b[39mwrite(response\u001b[38;5;241m.\u001b[39mcontent)\n", - "File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:189\u001b[0m, in \u001b[0;36mConnection.get\u001b[0;34m(self, query_options)\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, wait_time \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwait_seconds):\n\u001b[1;32m 188\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtry: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mi\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, wait_time: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mwait_time\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 189\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwait_and_get\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstatus_endpoint\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mwait_secs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mwait_time\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 190\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response\u001b[38;5;241m.\u001b[39mheaders[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mContent-Type\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mapplication/zip\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 191\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n", - "File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:273\u001b[0m, in \u001b[0;36mwait_and_get\u001b[0;34m(self, endpoint, query_options, wait_secs, timeout)\u001b[0m\n\u001b[1;32m 257\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwait_and_get\u001b[39m(\n\u001b[1;32m 258\u001b[0m \u001b[38;5;28mself\u001b[39m, endpoint : \u001b[38;5;28mstr\u001b[39m, query_options : Dict \u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, wait_secs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, timeout\u001b[38;5;241m=\u001b[39m(\u001b[38;5;241m3.05\u001b[39m, \u001b[38;5;241m20\u001b[39m)\n\u001b[1;32m 259\u001b[0m ):\n\u001b[1;32m 260\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"accesses given endpoint\u001b[39;00m\n\u001b[1;32m 261\u001b[0m \n\u001b[1;32m 262\u001b[0m \u001b[38;5;124;03m Parameters:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 271\u001b[0m \u001b[38;5;124;03m timeout for the request.\u001b[39;00m\n\u001b[1;32m 272\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 273\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m wait_secs:\n\u001b[1;32m 274\u001b[0m time\u001b[38;5;241m.\u001b[39msleep(wait_secs)\n\u001b[1;32m 276\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mget(endpoint, params\u001b[38;5;241m=\u001b[39mquery_options, timeout\u001b[38;5;241m=\u001b[39mtimeout)\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], + "outputs": [], "source": [ "#CAVE: this cell runs about 30minutes per requested year\n", "#the processing is done on the server of the TOAR database.\n",