diff --git a/tests/produce_data.ipynb b/tests/produce_data.ipynb index 14d00e9942a2257bcbc428890b3f25050520f4a1..c604debdeb9a911ea9292f43e892c2517b47b4ca 100644 --- a/tests/produce_data.ipynb +++ b/tests/produce_data.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -18,41 +18,62 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "#creation of request.\n", "\n", - "Config = namedtuple(\"Config\", [\"grid\", \"time\", \"variables\", \"stats\"])\n", + "Config = namedtuple(\"Config\", [\"grid\", \"time\", \"variables\", \"stats\",\"moreOptions\"])\n", + "\n", + "#moreOptions is implemented as a dict to add additional arguments to the query to the REST API\n", + "#For example the field toar1_category with its possible values Urban, RuralLowElevation, RuralHighElevation and Unclassified can be added.\n", + "#see page 18 in https://toar-data.fz-juelich.de/sphinx/TOAR_UG_Vol03_Database/build/latex/toardatabase--userguide.pdf\n", "\n", "valid_data = Config(\n", " RegularGrid( lat_resolution=1.9, lon_resolution=2.5, ),\n", " TimeSample( start=dt(2014,1,1), end=dt(2019,12,31), sampling=\"daily\"),\n", " [\"mole_fraction_of_ozone_in_air\"],#variable name\n", - " [\"mean\"]# [\"dma8epax\"], # enable when finished\n", + " [\"mean\"],# [\"dma8epax\"], # enable when finished\n", + " {}\n", ")\n", "missing_data = Config(\n", " RegularGrid( lat_resolution=1.9, lon_resolution=2.5),\n", " TimeSample( start=dt(2000,1,1), end=dt(2013,12,31), sampling=\"daily\"),\n", " [\"mole_fraction_of_ozone_in_air\"],\n", - " [\"dma8epax\"],\n", + " [\"mean\"],\n", + " #[\"dma8epax\"],\n", + " {}\n", ")\n", "\n", "configs = {\n", - " \"test_ta\": valid_data\n", + " #\"test_ta\" : valid_data\n", + " \"test_ta2\" : missing_data\n", "}\n", "\n", "#testing access:\n", - "config = configs[\"test_ta\"]\n", - "config.grid" + "#config = configs[\"test_ta\"]\n", + "#config.grid" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load status endpoint from cache\n", + "try: 1, wait_time: 300\n", + "try: 2, wait_time: 300\n", + "try: 3, wait_time: 300\n", + "try: 4, wait_time: 300\n", + "try: 5, wait_time: 300\n" + ] + } + ], "source": [ "#CAVE: this cell runs about 30minutes per requested year\n", "#the processing is done on the server of the TOAR database.\n", @@ -71,6 +92,7 @@ " time=config.time,\n", " variables=config.variables,\n", " stats=config.stats\n", + " #**config.moreOptions\n", " )\n", "\n", " for dataset, metadata in zip(datasets, metadatas):\n", diff --git a/toargridding/toar_rest_client.py b/toargridding/toar_rest_client.py index 554960f25de4b457efbf69f4517a42cefb5e6b9f..2e5e54c5553f58af3d3d3de327350edbafe86574 100644 --- a/toargridding/toar_rest_client.py +++ b/toargridding/toar_rest_client.py @@ -339,15 +339,13 @@ class AnalysisService: # remove data where utc -> sun/local ? time conversion leads to dateshift newDates = metadata.time.as_datetime_index() if len(timeseries.columns) == len(newDates)+2: + print(f"Info: removed columns {timeseries.columns[0]} and {timeseries.columns[-1]} to match data range of {newDates[0]} to {newDates[-1]}") timeseries.drop(columns=[first, last], inplace=True) - print("Info: removed first and last column from retrieved timeseries") elif len(timeseries.columns) == len(newDates)+1: + print(f"Info: removed columns {timeseries.columns[-1]} to match data range of {newDates[0]} to {newDates[-1]}") timeseries.drop(columns=[last], inplace=True) - print("Info: removed last column from retrieved timeseries") else: raise RuntimeError(f"There is a mismatch in the timestamps...\nDownloaded:{timeseries.columns}\nFrom Metadata: {newDates}") - print(timeseries.columns) - print(newDates) timeseries.columns = newDates all_na = timeseries.isna().all(axis=1)