diff --git a/README.md b/README.md
index 1f6372b283dfa03df12cfb58cafd1346f058a524..135017adb4a6d54b60ad50a85649f6a7f383a82e 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ The request to the database also allows a statistical analysis of the requested
The mean and standard deviation of all stations within a cell are computed.
The tool handles the request to the database over the REST API and the subsequent processing.
-The results are provided as xarray objects for subsequent processing by the user.
+The results of the gridding are provided as xarray objects for subsequent processing by the user.
This project is in beta with the intended basic functionalities.
The documentation is work in progress.
@@ -22,6 +22,7 @@ TBD, see pyproject.toml
Move to the folder you want to create download this project to.
We now need to download the source code (https://gitlab.jsc.fz-juelich.de/esde/toar-public/toargridding/-/tree/dev?ref_type=heads). Either as ZIP folder or via git:
+## Download with GIT
Clone the project from its git repository:
```
git clone https://gitlab.jsc.fz-juelich.de/esde/toar-public/toargridding.git
@@ -32,6 +33,7 @@ cd toargridding
git checkout dev
```
+## Installing Dependencies
The handling of required packages is done with poetry. So run poetry in the project directory:
```
@@ -40,30 +42,67 @@ poetry install
# Example
-There are at the moment three example provided as jupyter notebooks (https://jupyter.org/):
+There are at the moment three example provided as jupyter notebooks (https://jupyter.org/).
+
+Running them with the python environment produced by poetry can be done by
+```
+poetry run jupyter notebook
+```
+
## High level function
```
tests/produce_data.ipynb
```
Provides an example on how to download data, apply gridding and save the results as netCDF files.
+A possible improvement for is the exchange of the AnalysisService with AnalysisServiceDownload, which caches requests from the TOARDB.
+This allows different griddings without the necessity to repeat the request to the TOARDB and subsequent download.
+
+The example uses a dictionary to pass additional arguments to the request to the TAOR database.
+A detailed list can be found at https://toar-data.fz-juelich.de/api/v2/#stationmeta
+
+## Retrieving data
+```
+tests/get_sample_data.ipynb
+```
+Downloads data from the TOAR database. The extracted data are written to disc. No further processing or gridding is done.
+
## Retrieving data
-get_sample_data.ipynb
-Downloads data from the TOAR database.
+```
+tests/get_sample_data_manual.ipynb
+```
+Downloads data from the TOAR database with a manual creation of the request to the TOAR database.
+This example does not perform any gridding.
+
## Retriving data and visualization
```
-quality_controll.ipynb
+tests/quality_controll.ipynb
```
Notebook for downloading and visualization of data.
The data are downloaded and reused for subsequent executions of this notebook.
+The gridding is done on the downloaded data. Gridded data are not saved to disc.
+
+## Benchmarks Requests to TOAR Database
+
+```
+tests/benchmark.py
+```
+This script requests datasets with different durations (days to month) from the TOAR Database and saves them to disc.
+It reports the duration for the different requests.
+There is no gridding involved.
+CAVE: This script can run several hours.
+
# Supported Grids
-The first supported grid is the Cartesian grid.
+The first supported grid is a regular grid with longitude and latitude.
# Supported Variables
At the moment only a limited number of variables from the TOAR database is supported.
+# Supported Time intervals
+
+At the moment time differences larger than one day are working, i.e. start and end=start+1day leads to crashes.
# Documentation of Source Code:
diff --git a/tests/get_sample_data.ipynb b/tests/get_sample_data.ipynb
index fd3026b97d8e66c810691733e71b357e920a8cbf..35bc0f2ee29494ddc7a9a74a24d1a18f4c4fc83d 100644
--- a/tests/get_sample_data.ipynb
+++ b/tests/get_sample_data.ipynb
@@ -2,84 +2,71 @@
"cells": [
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 1,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2024-05-07 16:46:21.140327\n"
+ ]
+ }
+ ],
"source": [
- "from datetime import datetime\n",
+ "from datetime import datetime, timedelta\n",
"\n",
- "#samping monthly or daily\n",
- "sampling = \"monthly\"\n",
- "start = datetime(2010,1,1)\n",
- "end = datetime(2011,1,1)\n",
+ "from toargridding.metadata import TimeSample, Metadata\n",
"\n",
- "print(start.date(), end.date())\n",
- "print(start.isoformat(), end.isoformat())"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import requests\n",
+ "sampling = \"daily\" # FIXME check monthly !!!\n",
+ "start = datetime(2010, 1, 1)\n",
+ "end = datetime(2011, 1, 1)\n",
"\n",
- "response = requests.get(\n",
- " \"https://toar-data.fz-juelich.de/api/v2/analysis/statistics/\",\n",
- " params={\n",
- " \"daterange\": f\"{start.isoformat()},{end.isoformat()}\", # 1-year\n",
- " \"variable_id\": 5,\n",
- " \"statistics\": \"mean\",\n",
- " \"sampling\": sampling, # daily sampling\n",
- " \"min_data_capture\": 0,\n",
- " \"limit\": \"None\", # get all timeseries\n",
- " \"format\": \"by_statistic\",\n",
- " \"metadata_scheme\": \"basic\"\n",
- " }\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "print(response.status_code)\n",
- "status_endpoint = response.json()[\"status\"]\n",
- "print(status_endpoint)"
+ "statistics_endpoint = \"https://toar-data.fz-juelich.de/api/v2/analysis/statistics/\"\n",
+ "statistic = \"mean\"\n",
+ "\n",
+ "time = TimeSample(start, end, sampling=sampling)\n",
+ "# { \"station_type_of_area\" : \"urban\" } category is not known\n",
+ "metadata = Metadata.construct(\"mole_fraction_of_ozone_in_air\", time, statistic, { \"toar1_category\" : \"RuralLowElevation\"})#\n",
+ "\n",
+ "start_time = datetime.now()\n",
+ "print(start_time)"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2010-01-01 00:00:00\n",
+ "2011-01-01 00:00:00\n",
+ "Info: removed columns 2011-01-02 to match data range of 2010-01-01 00:00:00 to 2011-01-01 00:00:00\n",
+ "0:00:00.036132\n"
+ ]
+ }
+ ],
"source": [
- "#this cell can run 30minutes or longer.\n",
- "import time\n",
+ "from pathlib import Path\n",
+ "from toargridding.toar_rest_client import AnalysisServiceDownload\n",
"\n",
- "waiting_for_data = True\n",
"\n",
- "start_time = time.time()\n",
- "while waiting_for_data:\n",
- " time.sleep(30)\n",
- " response = requests.get(status_endpoint)\n",
+ "#creation of output directories\n",
+ "toargridding_base_path = Path(\".\")\n",
+ "cache_dir = toargridding_base_path / \"results\"\n",
+ "download_dir = toargridding_base_path / \"data\"\n",
+ "cache_dir.mkdir(parents=True, exist_ok=True)\n",
+ "download_dir.mkdir(parents=True, exist_ok=True)\n",
"\n",
- " waiting_for_data = (response.headers[\"Content-Type\"] == \"application/json\")\n",
+ "analysis_service = AnalysisServiceDownload(statistics_endpoint, cache_dir, download_dir)\n",
"\n",
- "response_time = time.time() - start_time\n",
- "print(response_time)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "response.headers[\"Content-Type\"], response.headers[\"Content-Length\"]"
+ "results = analysis_service.get_data(metadata)\n",
+ "\n",
+ "end_time = datetime.now()\n",
+ "print(end_time-start_time)"
]
},
{
@@ -87,15 +74,7 @@
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": [
- "from pathlib import Path\n",
- "outDir = Path(\"data\")\n",
- "outDir.mkdir(parents=False, exist_ok=True)\n",
- "fn = outDir / f\"{sampling}_{start.date()}_{end.date()}.zip\"\n",
- "with open(fn, \"w+b\") as sample_file:\n",
- " sample_file.write(response.content)\n",
- "print(f\"Wrote outout to file [path/to/toargridding]/tests/{fn}\")"
- ]
+ "source": []
}
],
"metadata": {
diff --git a/tests/produce_data.ipynb b/tests/produce_data.ipynb
index 14d00e9942a2257bcbc428890b3f25050520f4a1..b9d0f9c20a3448e22beb388feefff0b8cfd55a86 100644
--- a/tests/produce_data.ipynb
+++ b/tests/produce_data.ipynb
@@ -10,7 +10,7 @@
"from collections import namedtuple\n",
"from pathlib import Path\n",
"\n",
- "from toargridding.toar_rest_client import AnalysisService\n",
+ "from toargridding.toar_rest_client import AnalysisServiceDownload\n",
"from toargridding.grids import RegularGrid\n",
"from toargridding.gridding import get_gridded_toar_data\n",
"from toargridding.metadata import TimeSample"
@@ -24,23 +24,26 @@
"source": [
"#creation of request.\n",
"\n",
- "Config = namedtuple(\"Config\", [\"grid\", \"time\", \"variables\", \"stats\"])\n",
+ "Config = namedtuple(\"Config\", [\"grid\", \"time\", \"variables\", \"stats\",\"moreOptions\"])\n",
+ "\n",
+ "#moreOptions is implemented as a dict to add additional arguments to the query to the REST API\n",
+ "#For example the field toar1_category with its possible values Urban, RuralLowElevation, RuralHighElevation and Unclassified can be added.\n",
+ "#see page 18 in https://toar-data.fz-juelich.de/sphinx/TOAR_UG_Vol03_Database/build/latex/toardatabase--userguide.pdf\n",
+ "\n",
+ "details4Query ={\n",
+ " #\"toar1_category\" : \"Urban\" #uncomment if wished:-)\n",
+ "}\n",
"\n",
"valid_data = Config(\n",
" RegularGrid( lat_resolution=1.9, lon_resolution=2.5, ),\n",
- " TimeSample( start=dt(2014,1,1), end=dt(2019,12,31), sampling=\"daily\"),\n",
+ " TimeSample( start=dt(2000,1,1), end=dt(2019,12,31), sampling=\"daily\"),#possibly adopt range:-)\n",
" [\"mole_fraction_of_ozone_in_air\"],#variable name\n",
- " [\"mean\"]# [\"dma8epax\"], # enable when finished\n",
- ")\n",
- "missing_data = Config(\n",
- " RegularGrid( lat_resolution=1.9, lon_resolution=2.5),\n",
- " TimeSample( start=dt(2000,1,1), end=dt(2013,12,31), sampling=\"daily\"),\n",
- " [\"mole_fraction_of_ozone_in_air\"],\n",
- " [\"dma8epax\"],\n",
+ " [\"dma8epax\", \"mean\" ],# will start one request after another other...\n",
+ " details4Query\n",
")\n",
"\n",
"configs = {\n",
- " \"test_ta\": valid_data\n",
+ " \"test_ta\" : valid_data\n",
"}\n",
"\n",
"#testing access:\n",
@@ -60,9 +63,11 @@
"# The download can also take a few minutes\n",
"\n",
"stats_endpoint = \"https://toar-data.fz-juelich.de/api/v2/analysis/statistics/\"\n",
+ "cache_basepath = Path(\"cache\")\n",
"result_basepath = Path(\"results\")\n",
+ "cache_basepath.mkdir(exist_ok=True)\n",
"result_basepath.mkdir(exist_ok=True)\n",
- "analysis_service = AnalysisService(stats_endpoint, result_basepath)\n",
+ "analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath)\n",
"\n",
"for person, config in configs.items():\n",
" datasets, metadatas = get_gridded_toar_data(\n",
@@ -71,6 +76,7 @@
" time=config.time,\n",
" variables=config.variables,\n",
" stats=config.stats\n",
+ " #**config.moreOptions\n",
" )\n",
"\n",
" for dataset, metadata in zip(datasets, metadatas):\n",
diff --git a/tests/quality_controll.ipynb b/tests/quality_controll.ipynb
index ddceef5fafdf22fd0957666c72818bdb8b394f7f..744629b6c82b765395457ecc516619fb65765e70 100644
--- a/tests/quality_controll.ipynb
+++ b/tests/quality_controll.ipynb
@@ -39,8 +39,8 @@
"analysis_service = AnalysisServiceDownload(endpoint, cache_dir, data_download_dir)\n",
"my_grid = RegularGrid(1.9, 2.5)\n",
"\n",
- "time = TimeSample(dt(2016,1,1), dt(2016,12,31), \"daily\")\n",
- "metadata = Metadata.construct(\"mole_fraction_of_ozone_in_air\", \"mean\", time)\n"
+ "time = TimeSample(dt(2016,1,1), dt(2016,12,31), \"monthly\")\n",
+ "metadata = Metadata.construct(\"mole_fraction_of_ozone_in_air\", time, \"mean\")\n"
]
},
{
diff --git a/toargridding/gridding.py b/toargridding/gridding.py
index 3268dc267c8ca3fd1c00e0c33c59de529ad6c9e4..5523c9e84779adacfcbe1967c039e12f654d99bc 100644
--- a/toargridding/gridding.py
+++ b/toargridding/gridding.py
@@ -16,9 +16,32 @@ def get_gridded_toar_data(
time: TimeSample,
variables: list[str],
stats: list[str],
+ **kwargs
) -> tuple[list[xr.Dataset], list[Metadata]]:
+ """ API to download data as xarrays
+
+ The function creates all combinations of the variable and stats list
+
+ Parameters:
+ ----------
+ analysis_service:
+ access to the REST API and manages download of data
+ grid:
+ grid for the output data.
+ time:
+ sampled values in time
+ variables:
+ list of variables to be extracted from the TOARdatabase
+ stats:
+ list of statistical properties to be extracted from the TOAR database
+
+ return:
+ -------
+ Gridded datasets for each combination of variables and stats and appropriate metadata for each dataset.
+ """
+
metadatas = [
- Metadata.construct(var, stat, time) for var, stat in product(variables, stats)
+ Metadata.construct(standart_name=var, time=time, stat=stat, moreOptions=kwargs) for var, stat in product(variables, stats)
]
datasets = []
@@ -27,4 +50,5 @@ def get_gridded_toar_data(
ds = grid.as_xarray(data)
datasets.append(ds)
+ #TODO: return this as a list of tuples to keep data and metadata together?
return datasets, metadatas
diff --git a/toargridding/grids.py b/toargridding/grids.py
index d8f894938d5aa89dd7ef9aeb422f14f0a03959a4..83eb164ca5e264a4d202179102f37050fd5ce29e 100644
--- a/toargridding/grids.py
+++ b/toargridding/grids.py
@@ -6,6 +6,9 @@ import xarray as xr
import pandas as pd
import numpy as np
+from typing import Dict
+from pandas.core.groupby import DataFrameGroupBy
+
from toargridding.metadata import (
Variables,
Coordinates,
@@ -22,6 +25,8 @@ GridType = Enum("GridType", ["regular"])
class GridDefinition(ABC):
"""factory and base class for definition of different grids
+ usage: GridDefinition.construct( GridType, dict( parameter : value ) )
+ The dict must contain all parameters required for the creation of the desired GridType
"""
cell_index_name = "cell_index"
@@ -31,6 +36,11 @@ class GridDefinition(ABC):
@staticmethod
def construct(grid_type: GridType, **kwargs):
+ """creation of requested grid type
+
+ usage: GridDefinition.construct( GridType, dict( parameter : value ) )
+ The dict must contain all parameters required for the creation of the desired GridType
+ """
match (grid_type):
case GridType.regular:
return RegularGrid(**kwargs)
@@ -39,7 +49,7 @@ class GridDefinition(ABC):
@property
@abstractmethod
- def description(self):
+ def description(self)->str:
"""description of this grid
"""
pass
@@ -65,11 +75,32 @@ class GridDefinition(ABC):
class RegularGrid(GridDefinition):
"""definition of a regular grid with longitude and latitude.
+
+ The grid covers the complete globe and is defined by providing resolution for latitude (lat_resolution) and longitude (lon_resolution)
+
+ Argument:
+ --------
+ lat:
+ latitude coordinate axis.
+ lon:
+ longitude coordinate axis.
+ dims:
+ names of the dimensions of the data
"""
Coord = namedtuple("Coord", ["lat", "lon"])
def __init__(self, lat_resolution, lon_resolution):
+ """constructor from resolutions
+
+ Parameters:
+ ----------
+ lat_resolution:
+ resolution for latitude in degree
+ lon_resolution:
+ resolution for longitude in degree
+ """
+
super().__init__()
# TODO make sure only sensible resolutions
@@ -93,10 +124,23 @@ class RegularGrid(GridDefinition):
self._as_i_index = np.arange(spatial_size).reshape(spatial_shape).T
@property
- def description(self):
+ def description(self)->str:
+ """get description of grid
+ """
+
return f"regular global grid with lat/lon resolutions ({self.lat.step}, {self.lon.step})"
def as_xarray(self, data: AnalysisRequestResult) -> xr.Dataset:
+ """gridding of a request to the TOAR database
+
+ groups the stations into the cells of the grid and calculates mean and standard deviation for each cell.
+
+ Parameters:
+ ----------
+ data:
+ results of the request, including data, station coordinates and metadata of request
+ """
+
data_grouped_by_cell = self.group_data_by_cell(
data.stations_data, data.stations_coords
)
@@ -105,7 +149,19 @@ class RegularGrid(GridDefinition):
return dataset
- def group_data_by_cell(self, data: pd.DataFrame, coords: pd.DataFrame):
+ def group_data_by_cell(self, data: pd.DataFrame, coords: pd.DataFrame) -> DataFrameGroupBy:
+ """grouping of stations into cells
+
+ This function converts the lat/lon coordinates of the stations into cell indices and groups stations belonging to one cell.
+
+ Parameters:
+ ----------
+ data:
+ station data
+ coords:
+ station coordinates
+ """
+
cell_indices = self.as_cell_index(coords)
# will convert cell_indices to float as some nans ar present
@@ -115,7 +171,17 @@ class RegularGrid(GridDefinition):
return data_with_indices.groupby(GridDefinition.cell_index_name)
- def get_cell_statistics(self, groups) -> dict[Variables, pd.DataFrame]:
+ def get_cell_statistics(self, groups : DataFrameGroupBy) -> dict[str, pd.DataFrame]:
+ """calculation of mean, std and number of stations per cell
+
+ Parameters:
+ ----------
+ groups:
+ time series data grouped by stations in a cell
+ return:
+ dictionary with calculated quantities
+ """
+
stats = {
Variables.mean: groups.mean(),
Variables.std: groups.std(),
@@ -124,7 +190,19 @@ class RegularGrid(GridDefinition):
return stats
- def create_dataset(self, cell_statistics: pd.DataFrame, metadata: Metadata):
+ def create_dataset(self, cell_statistics : Dict, metadata: Metadata) -> xr.Dataset:
+ """creation of data set and filling with results from the gridding
+
+ Parameters:
+ ----------
+ cell_statistics:
+ values obtained for each cell. In the beginning these are mean, std and number of stations
+ metadata:
+ metadata of the request
+ return:
+ xarray dataset with coordinates and variables following the CF convention
+ """
+
time = Coordinate.from_data(
metadata.time.as_cf_index(),
Coordinates.time,
@@ -141,12 +219,39 @@ class RegularGrid(GridDefinition):
return gridded_ds
- def get_data_array_dict(self, time, aggregated_data, variable, metadata):
+ def get_data_array_dict(self, time : Coordinate, aggregated_data : pd.DataFrame, variable : Variables, metadata : Metadata) -> Dict[str, xr.DataArray]:
+ """conversion of data to a dict for assigning them to the Dataset
+
+ Parameters:
+ ----------
+ time:
+ temporal coordinate
+ aggregated_data:
+ obtained results per grid cell
+ variable:
+ variable to be added. This allows access to Cf conform metadata
+ metadata:
+ metadata of request
+ """
+
gridded_data = self.create_gridded_data(time, aggregated_data)
gridded_variable = Variable.from_data(gridded_data, variable, metadata)
return {variable.name: gridded_variable.as_data_array(self.dims)}
- def create_gridded_data(self, time, grouped_timeseries):
+ def create_gridded_data(self, time : Coordinate, grouped_timeseries : pd.DataFrame)->np.array:
+ """converts the available cell data to a full lat/lon-temporal data cube.
+
+ Parameters:
+ ----------
+ time:
+ temporal coordinate
+ grouped_timeseries:
+ data frame with station position and data
+ return:
+ 3D-array with axis time, latitude and longitude. Fields without data are nan (fill_value defined in GridDefinition init)
+ """
+
+ #CAVE: This function might involve black magic...
values = np.empty((time.size, self.lat.size, self.lon.size))
values[...] = self.fill_value
@@ -157,7 +262,10 @@ class RegularGrid(GridDefinition):
return values
- def as_cell_index(self, coords):
+ def as_cell_index(self, coords : pd.DataFrame) -> pd.Series:
+ """converts coordinates of stations into x and y indices of the regular grid
+ """
+
id_x = self.coord_to_index(coords[self.lat.name], self.lat.min, self.lat.step)
id_y = self.coord_to_index(coords[self.lon.name], self.lon.min, self.lon.step)
@@ -165,12 +273,35 @@ class RegularGrid(GridDefinition):
return pd.Series(id_i, index=id_x.index)
- def coord_to_index(self, coord, x0_axis, d_axis):
+ def coord_to_index(self, coord : pd.Series, x0_axis : float, d_axis : float) -> np.array:
+ """converts a coordinate into a bin index on one axis
+
+ Parameters:
+ ----------
+ coord:
+ coordinate for conversion
+ x0_axis:
+ offset of the axis
+ d_axis:
+ resolution of the axis
+ """
+
return (np.ceil((coord / d_axis) - 0.5) - x0_axis / d_axis).astype(int)
- def get_empty_grid(
- self, time: Variable, metadata: pd.DataFrame
- ) -> xr.Dataset: # TODO make CF-compliant => docs
+ def get_empty_grid(self, time: Variable, metadata: Metadata) -> xr.Dataset: # TODO make CF-compliant => docs
+ """creation of an empty dataset without data
+
+ Sets up a dataset with its three axis: time, longitude and latitude.
+ Adds global metadata to the dataset.
+
+ Parameters:
+ ----------
+ time:
+ temporal coordinate
+ Metadata:
+ information on request
+ """
+
coords = {
Variables.time.name: time.as_data_array(),
Variables.latitude.name: self.lat.as_data_array(),
diff --git a/toargridding/metadata.py b/toargridding/metadata.py
index abf93492ec67156b4ad17e45f49e47a253bc0c70..6ace341523dda3f8738f35d491656e3ab5a6c39e 100644
--- a/toargridding/metadata.py
+++ b/toargridding/metadata.py
@@ -1,6 +1,6 @@
from datetime import datetime, timedelta
from enum import Enum
-from dataclasses import dataclass
+from dataclasses import dataclass, field
import numpy as np
import pandas as pd
@@ -11,6 +11,7 @@ from toargridding.static_metadata import global_cf_attributes, TOARVariable
from typing import Dict
date_created = datetime.utcnow().strftime("%Y-%m-dT%H:%M:%SZ")
+#date_created = datetime.now(datetime.UTC).strftime("%Y-%m-dT%H:%M:%SZ") # fix as utcnow will be removed in the future
COORDINATE_VARIABLES = ["latitude", "longitude", "time"]
DATA_VARIABLES = ["mean", "std", "n"]
@@ -28,6 +29,14 @@ class TimeSample:
"""Sampling in time
provides conversion into different formats
+
+ Attributes:
+ start:
+ start time point
+ end:
+ end time point
+ sampling:
+ temporal aggregation of values, e.g. daily, monthly
"""
start: datetime
@@ -35,7 +44,7 @@ class TimeSample:
sampling: str
@property
- def sampling(self) -> str: # TODO make better
+ def sampling(self) -> str:
"""sampling for data request
Sampling, i.e. the period used for the calculation of a parameters within the TOAD DB
@@ -46,19 +55,21 @@ class TimeSample:
@sampling.setter
def sampling(self, sampling : str):
if sampling not in ALLOWED_SAMPLING_VALUES:
- raise ValueError(f"sampling: {sampling} is not in the list of supported samplings for toargridding.")
+ raise ValueError(f"sampling: {sampling} is not in the list of supported samplings for toargridding: {ALLOWED_SAMPLING_VALUES}")
self._sampling = sampling
def as_datetime_index(self) -> pd.DatetimeIndex:
"""Conversion to array with all sampled time points
"""
+ print(self.start)
+ print(self.end)
return pd.period_range(self.start, self.end, freq=self.frequency).to_timestamp()
@property
def daterange_option(self) -> str:
"""Conversion of range to a string
- Range is given as start<=incuded days < end
+ Range is given as [start, end)
"""
end_with_padding = self.end + timedelta(1)
return f"{self.start.isoformat()},{end_with_padding.isoformat()}"
@@ -74,33 +85,64 @@ class TimeSample:
Calculates the duration in days relative to start time point.
"""
- n_days = (self.end - self.start).days
- return np.arange(n_days + 1)
+
+ tp = self.as_datetime_index()
+ return np.array([ (t - tp[0]).days for t in tp])
@dataclass
class Metadata:
- """MEtadata of a request.
-
+ """Metadata of a request.
+
+ Attributes:
+ ----------
variable:
support variable of the TOAR data base
- statistics:
- statistical processing applied by the TOAR database for this request
time:
requested time points
+ statistics:
+ statistical processing applied by the TOAR database for this request
+ moreOptions:
+ collection of additional query options for the REST API.
"""
variable: TOARVariable
- statistic: str
time: TimeSample
+ statistic: str
+ moreOptions : Dict = field(default_factory=lambda: {})
@staticmethod
- def construct(standart_name: str, stat: str, time: TimeSample):
+ def construct(standart_name: str, time: TimeSample, stat: str, moreOptions : Dict = {}):
+ """constructor
+
+ Parameters:
+ ----------
+ standart_name:
+ standard name according to CF
+ stat:
+ statistical analysis to be done by the TOAR database
+ time:
+ temporal sampling of this request
+ moreOptions:
+ collection of additional query options for the REST API.
+ """
+
variable = TOARVariable.get(standart_name)
+ return Metadata(variable, time, stat, moreOptions)
+
+ @property
+ def statistic(self) -> str: # TODO make better
+ """statistical property for being extracted from the TOAR database
+ This can for example be the mean, max, min, median. For a full list see toargridding.toarstat_constants.STATISTICS_LIST
+
+ """
+ return self._statistic
+
+ @statistic.setter
+ def statistic(self, stat : str):
if stat not in STATISTICS_LIST:
raise ValueError(f"invalid statistic: {stat}")
-
- return Metadata(variable, stat, time)
+ self._statistic = stat
def get_id(self) -> str:
"""creation of a request specific ID
@@ -171,6 +213,7 @@ def get_cf_metadata(variable: Variables, metadata: Metadata | None)-> Dict:
The resulting dictionary contains the required values for the netCDF files.
"""
+
match variable.name:
case Variables.latitude.name:
cf_metadata = {
diff --git a/toargridding/static_metadata.py b/toargridding/static_metadata.py
index 66e3f28062be193d682a54dd8b3577c80280f1ae..adf6566a301795d1e71f90c9d983442f2fc36137 100644
--- a/toargridding/static_metadata.py
+++ b/toargridding/static_metadata.py
@@ -25,7 +25,7 @@ class TOARVariable:
"""
vars = None
- """available TOAR variables."""
+ """available variables from the TOAR database."""
name: str
longname: str
@@ -49,8 +49,14 @@ class TOARVariable:
@classmethod
def get(cls, standart_name: str) -> "TOARVariable":
"""searches the known variables for the requested variable
+
+ Parameters:
+ ----------
+ standart_name:
+ name of the variable following the CF convention.
- return: provides direct access to the meta data of the selected variable
+ return:
+ provides direct access to the meta data of the selected variable
"""
for var in cls.vars:
if var.standart_name == standart_name:
diff --git a/toargridding/toar_rest_client.py b/toargridding/toar_rest_client.py
index a1f75ae5592ed3de1578409581170635edd3e5c4..7c2406787e9190824af61d8021c611098e9d89f2 100644
--- a/toargridding/toar_rest_client.py
+++ b/toargridding/toar_rest_client.py
@@ -1,7 +1,7 @@
import time
import io
from zipfile import ZipFile
-from dataclasses import dataclass, asdict
+from dataclasses import dataclass, asdict, field
from contextlib import contextmanager
import json
from pathlib import Path
@@ -9,6 +9,8 @@ from pathlib import Path
import requests
import pandas as pd
+from typing import Dict
+
from toargridding.metadata import Metadata, AnalysisRequestResult, Coordinates
@@ -20,7 +22,29 @@ COORDS = [STATION_LAT, STATION_LON]
@dataclass(frozen=True)
class QueryOptions:
"""Creation of a request to the TOAR database.
+
+ Attributes:
+ ----------
+ datarange:
+ extraction of timestamps with start<=valid<end. Format: [start],[end] with start and end in isoformat
+ variable_id:
+ id of the variable within the TOAR database
+ statistics:
+ statistical quantity requested from the TOAR database. see toargridding.toarstats_constants.STATISTICS_LIST.
+ sampling:
+ temporal aggregation, e.g. daily, monthly
+ min_data_capture:
+ most probably the minimum data to include in the request
+ metadata_schema:
+ amount of metadata being provided, see Quick Start for TOAR Analysis Service
+ limit:
+ limit to amount of extracted data; see Quick Start for TOAR Analysis Service
+ format:
+ output format; see Quick Start for TOAR Analysis Service
+ moreOptions:
+ dict with additional query options for the request to the TOAR database.
"""
+
daterange: str
variable_id: str
statistics: str
@@ -29,28 +53,62 @@ class QueryOptions:
metadata_scheme: str = "basic"
limit: str = "None"
format: str = "by_statistic"
+ moreOptions : Dict = field(default_factory=lambda: {}) #needs to be last element for to dict factory
@staticmethod
def from_metadata(metadata: Metadata):
"""Creation from Metadata object
+ Copies datarange, variable_id, statistics and sampling from the metadata object.
+ For the other parameters the default values are used.
"""
return QueryOptions(
daterange=metadata.time.daterange_option,
variable_id=str(metadata.variable.toar_id),
statistics=metadata.statistic,
sampling=metadata.time.sampling,
+ moreOptions=metadata.moreOptions
)
@property
def cache_key(self):
"""creation to identify the request in the cache of known request.
"""
- return "".join(asdict(self).values())
+ return "".join(asdict(self, dict_factory=quarryToDict).values())
+def quarryToDict(data : QueryOptions):
+ #print(data)
+ out = { field : value for field, value in data[:-1] }
+ extraVals = data[-1][1]
+ for field, value in extraVals.items():
+ if not field in data:
+ out[field] = value
+ else:
+ raise ValueError(f"Providing invalid value for TAOR database: {field} is controlled by Metadata class")
+ return out
+
class Cache:
+ """cache to store download links for requests to the TOAD database
+
+ The extraction of data and statistical processing by the TOAR database can take hours or even days, depending on the amount of requested data.
+ The cache is persistent to allow the access to data after turning of the computer.
+
+ It is created in a dict like way and supports operations like "in".
+ A textfile called status_endpoints.json is created in cache_dir. CAVE: The first entry is required for loading the empty file.
+
+ """
+
def __init__(self, cache_dir : Path):
+ """constructor
+
+ Throws exception if cache directory does not exists.
+ Parameters
+ ----------
+ cache_dir:
+ directory for storing cache file.
+ """
+
if not cache_dir.exists():
raise RuntimeError(f"Given directory for saving cache file does not exists. Path: {cache_dir}")
self.cache_file = cache_dir / "status_endpoints.json"
@@ -60,18 +118,26 @@ class Cache:
json.dump({"foo": "bar"}, cache)
def __contains__(self, item: str):
+ """allows usage of "in"
+ """
with self.storage_dict() as storage:
return item in storage.keys()
def get(self, key: str):
+ """get an endpoint from the cache.
+ """
with self.storage_dict() as storage:
return storage[key]
def put(self, key: str, content: str):
+ """get add key and content as key-value-pair to cache
+ """
with self.storage_dict() as storage:
storage[key] = content
def remove(self, key: str):
+ """remove a key and content as key-value-pair to cache
+ """
with self.storage_dict() as storage:
del storage[key]
@@ -88,12 +154,33 @@ class Cache:
class Connection:
def __init__(self, endpoint, cache_dir):
+ """connection to the rest API of the TOAR database
+
+ This class handles the creation of requests and the interaction with the TOAR database.
+ It relies on the "TOARDB Analysis FastAPI REST interface"
+ https://toar-data.fz-juelich.de/api/v2/analysis/
+ with the default endpoint: https://toar-data.fz-juelich.de/api/v2/analysis/statistics
+
+ Parameters:
+ ----------
+ endpoint:
+ link to the TOAR database analysis service
+ cache_dir:
+ directory to store cache file
+ """
+
self.endpoint = endpoint
self.cache = Cache(cache_dir)
# max wait time is 30min
self.wait_seconds = [minutes * 60 for minutes in (5, 5, 5, 5, 5, 5)]
- def get(self, query_options):
+ def get(self, query_options : QueryOptions):
+ """get results for a request.
+
+ This is the main function to obtained data from the TOAR DB. It will start requests or lookup if an already started requests is finished.
+
+ Throws an exception, if the results are not available after the waiting time. A restart of the function continues the regular lookup for results.
+ """
status_endpoint = self.get_status_endpoint(query_options)
for i, wait_time in enumerate(self.wait_seconds):
@@ -107,6 +194,17 @@ class Connection:
)
def get_status_endpoint(self, query_options: QueryOptions):
+ """get endpoint to results of a request
+
+ This function checks if the request is already known and has been submitted to the TOAD DB.
+ If yes, the know endpoint is returned.
+ If the cache knows the endpoint, but the DB has deleted it, the endpoint is removed from the cache and a new request is started.
+ Otherwise a new new request is started.
+
+ Parameters:
+ ----------
+ Options for the request.
+ """
if query_options.cache_key in self.cache:
status_endpoint = self.cache.get(query_options.cache_key)
@@ -124,15 +222,40 @@ class Connection:
return status_endpoint
def query_for_status_endpoint(self, query_options: QueryOptions):
- response = self.wait_and_get(self.endpoint, asdict(query_options))
- status_endpoint = response.json()["status"]
+ """create and new request to the TOAR DB.
+
+ Adds the status endpoint of the request to the cache.
+
+ Parameters:
+ ----------
+ query_options:
+ request to the TOAR database.
+ """
+ response = self.wait_and_get(self.endpoint, asdict(query_options, dict_factory=quarryToDict))
+ try:
+ status_endpoint = response.json()["status"]
+ except:
+ raise RuntimeError(f"Request was not successful. Response by TOAR database: {response.text}")
self.cache.put(query_options.cache_key, status_endpoint)
return status_endpoint
def wait_and_get(
- self, endpoint, query_options=None, wait_secs=None, timeout=(3.05, 20)
+ self, endpoint : str, query_options : Dict =None, wait_secs=None, timeout=(3.05, 20)
):
+ """accesses given endpoint
+
+ Parameters:
+ ----------
+ endpoint:
+ either full endpoint of a request of base endpoint.
+ query_options:
+ used with the base endpoint to create a request. If None, endpoint is expected to be a full endpoint
+ wait_secs:
+ sleep in seconds before starting request to TAORDB
+ timeout:
+ timeout for the request.
+ """
if wait_secs:
time.sleep(wait_secs)
@@ -143,9 +266,31 @@ class AnalysisService:
METADATA = "metadata"
def __init__(self, stats_endpoint, cache_dir):
+ """constructor for setting up cache and connection
+
+ Parameters:
+ ---------
+ stats_endpoint:
+ link to statistics service of TOAR DB
+ cache_dir:
+ directory to store cache file for requests, needs to exist
+ """
self.connection = Connection(stats_endpoint, cache_dir)
def get_data(self, metadata: Metadata) -> AnalysisRequestResult:
+ """main function to obtain data from the TOAR DB
+
+ Handles requesting and loading of data into memory as soon as they are available.
+ In addition the data and coordinates undergo a cleanup.
+
+ Parameters:
+ ----------
+ metadata:
+ meta data for the request.
+ return:
+ Requested data and statistics, station coordinates and metadata of the request
+ """
+
timeseries, timeseries_metadata = self.get_timeseries_and_metadata(metadata)
coords = self.get_clean_coords(timeseries_metadata)
timeseries = self.get_clean_timeseries(timeseries, metadata)
@@ -154,23 +299,54 @@ class AnalysisService:
def get_timeseries_and_metadata(
self, metadata: Metadata
) -> tuple[pd.DataFrame, pd.DataFrame]:
+ """obtain data and metadata from TOAR database
+
+ return:
+ tuple[timeseries, station coordinates]
+ """
+
query_options = QueryOptions.from_metadata(metadata)
result = self.connection.get(query_options)
timeseries, timeseries_metadata = self.load_data(result.content, metadata)
return timeseries, timeseries_metadata
- def get_clean_coords(self, timeseries_metadata):
+ def get_clean_coords(self, timeseries_metadata : pd.DataFrame):
+ """remove all stations with invalid coordinates
+ invalid coordinates are NaN, none etc.
+ return:
+ stations with valid coordinates
+ """
coords = timeseries_metadata[COORDS]
coords.columns = [Coordinates.latitude.name, Coordinates.longitude.name]
valid_coords = coords.notna().all(axis=1)
return coords[valid_coords]
- def get_clean_timeseries(self, timeseries, metadata: Metadata):
+ def get_clean_timeseries(self, timeseries : pd.DataFrame, metadata: Metadata):
+ """replaces all nan in the data with 0 for plotting
+
+ timeseries:
+ extracted time series
+ metadata:
+ metadata belonging ot the timeseries.
+
+ return:
+ timeseries without invalid numbers (none, NaN, etc)
+ """
+ # TODO: Why are there different numbers of columns to be dropped??
# TODO maybe use cf-index here already ?
first, last = timeseries.columns[0], timeseries.columns[-1]
+ ##here we observe some differences in the number of timestamps.
# remove data where utc -> sun/local ? time conversion leads to dateshift
- timeseries.drop(columns=[first, last], inplace=True)
- timeseries.columns = metadata.time.as_datetime_index()
+ newDates = metadata.time.as_datetime_index()
+ if len(timeseries.columns) == len(newDates)+2:
+ print(f"Info: removed columns {timeseries.columns[0]} and {timeseries.columns[-1]} to match data range of {newDates[0]} to {newDates[-1]}")
+ timeseries.drop(columns=[first, last], inplace=True)
+ elif len(timeseries.columns) == len(newDates)+1:
+ print(f"Info: removed columns {timeseries.columns[-1]} to match data range of {newDates[0]} to {newDates[-1]}")
+ timeseries.drop(columns=[last], inplace=True)
+ else:
+ raise RuntimeError(f"There is a mismatch in the timestamps...\nDownloaded:{timeseries.columns}\nFrom Metadata: {newDates}")
+ timeseries.columns = newDates
all_na = timeseries.isna().all(axis=1)
timeseries = timeseries[~all_na]
@@ -181,6 +357,15 @@ class AnalysisService:
def load_data(
self, content: bytes, metadata: Metadata
) -> tuple[pd.DataFrame, pd.DataFrame]:
+ """convert downloaded byte stream into pandas dataframes
+
+ Parameters:
+ ----------
+ content:
+ downloaded data as zip file
+ metadata:
+ information on the request.
+ """
zip_stream = io.BytesIO(content)
with ZipFile(zip_stream) as myzip:
timeseries = self.extract_data(myzip, metadata.statistic)
@@ -188,13 +373,38 @@ class AnalysisService:
return timeseries, timeseries_metadata
- def extract_data(self, zip_file, data_file) -> pd.DataFrame:
+ def extract_data(self, zip_file : ZipFile, data_file : str) -> pd.DataFrame:
+ """extract a specific csv file from the zip file
+
+ Parameters:
+ ----------
+ zip_file:
+ opened zip file
+ data_file:
+ base file name of the requested file. Extension .csv is added by this function.
+ """
with zip_file.open(f"{data_file}.csv") as f:
s_stream = io.StringIO(f.read().decode("utf-8"))
return pd.read_csv(s_stream, comment="#", index_col=0)
class AnalysisServiceDownload(AnalysisService):
+ """download service with caching of requests to the TOARDB
+
+ This service performs the request to the TOAR database and downloads the results of the request to disc before returning if for further processing.
+ When retrieving data, a check is donw, if this request has already been cached on disc.
+
+ Attributes:
+ ----------
+ stats_endpoint:
+ link to statistics service of TOAR DB
+ cache_dir:
+ directory to store cache file for requests, needs to exist
+ sample_dir:
+ directory for caching results of request to the TOARDB
+ use_downloaded:
+ flag to control if the cache of downloaded requests is checked before extracting data from the TOARDB
+ """
def __init__(
self, stats_endpoint, cache_dir, sample_dir: Path, use_downloaded=True
):
@@ -202,7 +412,16 @@ class AnalysisServiceDownload(AnalysisService):
self.sample_dir = sample_dir
self.use_downloaded = use_downloaded
- def get_timeseries_and_metadata(self, metadata: Metadata):
+ def get_timeseries_and_metadata(self, metadata: Metadata) -> tuple[pd.DataFrame, pd.DataFrame]:
+ """loading of cached data or requesting of data from the TOARDB
+
+ Parameters:
+ ----------
+ metadata:
+ information on requested data
+ return:
+ tuple[timeseries, station coordinates]
+ """
filename = self.sample_dir / self.get_sample_file_name(metadata)
query_options = QueryOptions.from_metadata(metadata)
needs_fresh_download = (not self.use_downloaded) or (not filename.is_file())
@@ -220,4 +439,13 @@ class AnalysisServiceDownload(AnalysisService):
@staticmethod
def get_sample_file_name(metadata: Metadata):
- return f"{metadata.statistic}_{metadata.time.sampling}_{metadata.time.start.date()}_{metadata.time.end.date()}.zip"
+ """creates a filename from the metadata
+
+ At the moment considering statistical method, sampling (temporal aggregation) as well as start and end.
+ Parameters:
+ ----------
+ metadata:
+ metadata for the request.
+ """
+ addition = "_".join(metadata.moreOptions.values())
+ return f"{metadata.statistic}_{metadata.time.sampling}_{metadata.variable.cf_standardname}_{metadata.time.start.date()}_{metadata.time.end.date()}_{addition}.zip"
diff --git a/toargridding/variables.py b/toargridding/variables.py
index 3e29d8722d85bfae4ec1ebb28933a41fd64bb59a..5445cf543363e2f1228a9bebc0c452c9b22f2422 100644
--- a/toargridding/variables.py
+++ b/toargridding/variables.py
@@ -24,13 +24,14 @@ class Variable:
encoding:
encoding of data type
"""
+
var: Variables
data: np.ndarray
attributes: dict[str, str]
encoding: dict[str, str]
@classmethod
- def from_data(cls, data, variable: Variables, metadata: Metadata | None, **kwargs):
+ def from_data(cls, data : np.array, variable: Variables, metadata: Metadata | None, **kwargs):
"""construction from analysis results
"""
cf_metadata = get_cf_metadata(variable, metadata=metadata)
@@ -86,6 +87,18 @@ class Coordinate(Variable):
cls, variable: Variables, resolution: float, min: float, max: float
):
"""construction from a data range and resolution
+
+ Creates a coordinate axis between min and amx with a step size close to resolution.
+
+ Parameters:
+ ----------
+ resolution:
+ width of a bin; actual size will be selected to obtain equidistant steps between steps
+ min:
+ lowest value on coordinate axis
+ max:
+ highest value on coordinate axis
+
"""
span = max - min