diff --git a/README.md b/README.md
index 15f12d0c16de2e5446eb7de551ee9f682aa5692b..a4505e82747de814ced52ea6201196eb22fe55a1 100644
--- a/README.md
+++ b/README.md
@@ -50,9 +50,12 @@ For the installation of all required dependencies call
```bash
pip install -e .
```
+To be able to execute the examples, that are provided as jupyter notebooks, we need to install a different preset by calling
+```bash
+pip install -e . "interactive"
+```
-
-To run scripts or notebooks use:
+To run the example notebooks:
```bash
#for selecting a notebook over the file browser in your webbrowser:
jupyter notebook
diff --git a/examples/quality_controll.ipynb b/examples/quality_controll.ipynb
index 35e29fa5bce22277cbf0c75bb22e86b83444360a..abd87257a311323fbaaebeda4bc88a69bcab8c59 100644
--- a/examples/quality_controll.ipynb
+++ b/examples/quality_controll.ipynb
@@ -28,6 +28,8 @@
"from toargridding.metadata import Metadata, TimeSample, AnalysisRequestResult, Coordinates\n",
"from toargridding.variables import Coordinate\n",
"\n",
+ "from toargridding.contributors import contributionsManager\n",
+ "\n",
"import logging\n",
"from toargridding.defaultLogging import toargridding_defaultLogging\n",
"#setup of logging\n",
@@ -42,7 +44,7 @@
"cache_dir = toargridding_base_path / \"cache\"\n",
"data_download_dir = toargridding_base_path / \"results\"\n",
"\n",
- "analysis_service = AnalysisServiceDownload(endpoint, cache_dir, data_download_dir, use_downloaded=False)\n",
+ "analysis_service = AnalysisServiceDownload(endpoint, cache_dir, data_download_dir, use_downloaded=True)\n",
"my_grid = RegularGrid(1.9, 2.5)\n",
"\n",
"time = TimeSample(dt(2016,1,1), dt(2016,2,28), \"daily\")\n",
@@ -57,7 +59,15 @@
"source": [
"# this cell can runs longer than 30minutes\n",
"data = analysis_service.get_data(metadata)\n",
- "ds = my_grid.as_xarray(data)"
+ "\n",
+ "# create contributors endpoint and write result to metadata\n",
+ "contrib = contributionsManager(metadata.get_id(), data_download_dir)\n",
+ "contrib.extract_contributors_from_data_frame(data.stations_data)\n",
+ "metadata.contributors_metadata_field = contrib.setup_contributors_endpoint_for_metadata()\n",
+ "ds = my_grid.as_xarray(data)\n",
+ "print(ds)\n",
+ "#store dataset\n",
+ "ds.to_netcdf(data_download_dir / f\"{metadata.get_id()}_{my_grid.get_id()}.nc\")"
]
},
{
diff --git a/pyproject.toml b/pyproject.toml
index 51d4d7cc7a688866f2574db1390385a8a0760e77..527d939027bc410fca2f49b05a23ebe79770b7e7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -25,6 +25,7 @@ dependencies = [
"numpy",
"xarray",
"pandas",
+ "netCDF4",
]
[project.urls]
@@ -48,7 +49,7 @@ installer = "uv"
extra-dependencies = [
"jupyter",
"ipykernel",
- "cartopy"
+ "cartopy",
]
[tool.hatch.envs.hatch-static-analysis]
@@ -79,4 +80,4 @@ exclude_lines = [
"no cov",
"if __name__ == .__main__.:",
"if TYPE_CHECKING:",
-]
\ No newline at end of file
+]
diff --git a/src/toargridding/contributors.py b/src/toargridding/contributors.py
index 4bb85a9b9eae4d3659eb3e78ac6d293885ba973e..07ce3e70afd0af212f7b76a73ca5f748f16ce9a1 100644
--- a/src/toargridding/contributors.py
+++ b/src/toargridding/contributors.py
@@ -1,11 +1,12 @@
+from pandas import DataFrame
from pandas.core.groupby import DataFrameGroupBy
from typing import Iterable
from pathlib import Path
#TODO: maybe create an abstract base class and separate the offline and service implementations
class contributionsManager:
- def __init__(self, requestID, contributors_path : Path = None, endpoint=""):
+ def __init__(self, requestID, contributors_path : Path = None, endpoint="https://toar-data.fz-juelich.de/api/v2/request_contributors"):
self.requestID = requestID
self.timeseriesIDs = set()
self.endpoint = endpoint
@@ -32,7 +33,9 @@ class contributionsManager:
def add_timeseries_ids(self, ids : Iterable[int]) -> None:
for id in ids:
self.timeseriesIDs.add(id)
- def extract_contributors(self, data_grouped_by_cell : DataFrameGroupBy):
+ def extract_contributors_from_data_frame(self, data_frame : DataFrame):
+ self.add_timeseries_ids( data_frame.index.to_list() )
+ def extract_contributors_from_grouped_dataframe(self, data_grouped_by_cell : DataFrameGroupBy):
for _, table in data_grouped_by_cell:
self.add_timeseries_ids( table.index.to_list() )
diff --git a/src/toargridding/grids.py b/src/toargridding/grids.py
index 1e7a003e135cefdde19cae56b6d5feaa408218f6..f5e8180d4994a7ab4a1db37a0816c6104b5adecd 100644
--- a/src/toargridding/grids.py
+++ b/src/toargridding/grids.py
@@ -137,7 +137,6 @@ class RegularGrid(GridDefinition):
results of the request, including data, station coordinates and metadata of request
"""
data_grouped_by_cell = self.group_data_by_cell(data.stations_data, data.stations_coords)
- data.contributions.extract_contributors(data_grouped_by_cell)
cell_statistics = self.get_cell_statistics(data_grouped_by_cell)
dataset = self.create_dataset(cell_statistics, data.metadata)
@@ -182,7 +181,7 @@ class RegularGrid(GridDefinition):
return stats
- def create_dataset(self, cell_statistics: dict[str, pd.DataFrame], metadata: Metadata, contributions : contributionsManager) -> xr.Dataset:
+ def create_dataset(self, cell_statistics: dict[str, pd.DataFrame], metadata: Metadata) -> xr.Dataset:
"""creation of data set and filling with results from the gridding
Parameters:
diff --git a/src/toargridding/metadata.py b/src/toargridding/metadata.py
index d6ad9af8bfa6a887588e54dacb5ea814664c46e5..63ca02d5c491ba161e562bbbea9b490cd8507ba3 100644
--- a/src/toargridding/metadata.py
+++ b/src/toargridding/metadata.py
@@ -96,6 +96,8 @@ class TimeSample:
@dataclass
class Metadata:
"""Metadata of a request.
+ This class is responsible for the book keeping of all processing steps on the data.
+ This starts with the metadata used for the request and includes further aspects like filters that are applied.
Attributes:
----------
@@ -113,7 +115,6 @@ class Metadata:
time: TimeSample
statistic: str
moreOptions: dict = field(default_factory=dict)
- requestID : str = None
contributors_metadata_field : str = None
@staticmethod
diff --git a/src/toargridding/toar_rest_client.py b/src/toargridding/toar_rest_client.py
index 3561b8a34a3797ec47814f49ac7fb3dc8e0d38b0..57c7981d0e4cadbc5f7f578eb86fbddae1b8e6a3 100644
--- a/src/toargridding/toar_rest_client.py
+++ b/src/toargridding/toar_rest_client.py
@@ -168,7 +168,7 @@ class Cache:
"""! get now as string for saving in cache"""
return datetime.now().strftime(Cache.__timeFormat)
- def clearCache(self):
+ def clear_cache(self):
"""!Delete all values from the cache, that are older than the maxDaysInCache value"""
with self.storage_dict() as storage:
now = datetime.now()
@@ -184,7 +184,7 @@ class Cache:
del storage[key]
@staticmethod
- def setMaxDaysInCache(maxDays: float):
+ def set_max_days_in_cache(maxDays: float):
"""! set the maximum age of entries in the cache.
Also accepts fractions of a day down to seconds.
Parameters:
@@ -218,13 +218,13 @@ class Connection:
self.endpoint = endpoint
self.cache = Cache(cache_dir)
- self.cache.clearCache()
+ self.cache.clear_cache()
self.cache_backup = Cache(cache_dir, "status_endpoints.old")
# max wait time is 30min
self.wait_seconds = []
- self.setRequestTimes(5, 30)
+ self.set_request_times(5, 30)
- def setRequestTimes(self, interval_min, maxWait_min):
+ def set_request_times(self, interval_min, max_wait_minutes):
"""set the intervals and maximum duration to wait, before requests to the analysis service are stopped
The waiting intervals determine how long and often the status endpoint is checked if the results are available.
@@ -237,13 +237,13 @@ class Connection:
maxWait_min:
maximum duration to wait in minutes.
"""
- if maxWait_min <= 0:
+ if max_wait_minutes <= 0:
msg = "The maximum waiting time needs to be larger than 0min."
raise RuntimeError(msg)
- elif interval_min < 0 or interval_min > maxWait_min:
- self.wait_seconds[0] = maxWait_min
+ elif interval_min < 0 or interval_min > max_wait_minutes:
+ self.wait_seconds[0] = max_wait_minutes
else:
- self.wait_seconds = [interval_min * 60 for _ in range(interval_min, maxWait_min + 1, interval_min)]
+ self.wait_seconds = [interval_min * 60 for _ in range(interval_min, max_wait_minutes + 1, interval_min)]
def get(self, query_options: QueryOptions) -> requests.models.Response:
"""get results for a request.
@@ -419,18 +419,19 @@ class AnalysisService:
Handles requesting and loading of data into memory as soon as they are available.
In addition the data and coordinates undergo a cleanup.
+ This results in a datasets which only contain timeseries and station coordinates, if both data are present and valid, i.e. not nan.
+ A timeseries needs to have at least 1 non nan value to be considered as valid.
Parameters:
----------
metadata:
meta data for the request.
return:
- Requested data and statistics, station coordinates and metadata of the request
+ Requested data and statistics, station coordinates and metadata of the request.
"""
timeseries, timeseries_metadata = self.get_timeseries_and_metadata(metadata)
- coords = self.get_clean_coords(timeseries_metadata)
- timeseries = self.get_clean_timeseries(timeseries, metadata)
+ coords, timeseries = self.get_clean_timeseries_and_data(timeseries, timeseries_metadata, metadata )
return AnalysisRequestResult(timeseries, coords, metadata)
def get_timeseries_and_metadata(self, metadata: Metadata) -> tuple[pd.DataFrame, pd.DataFrame]:
@@ -442,10 +443,16 @@ class AnalysisService:
query_options = QueryOptions.from_metadata(metadata)
result = self.connection.get(query_options)
- print(result.request)
- print(result.request.url)
timeseries, timeseries_metadata = self.load_data(result.content, metadata)
return timeseries, timeseries_metadata
+
+ def get_clean_timeseries_and_data(self, timeseries : pd.DataFrame, timeseries_metadata : pd.DataFrame, metadata : Metadata) -> tuple[pd.DataFrame, pd.DataFrame]:
+ coords = self.get_clean_coords(timeseries_metadata)
+ timeseries = self.get_clean_timeseries(timeseries, metadata)
+
+ timeseries, coords = timeseries.align( coords, join="inner", axis=0 )
+ return coords, timeseries
+
def get_clean_coords(self, timeseries_metadata: pd.DataFrame):
"""remove all stations with invalid coordinates and drop unused metadata
@@ -579,8 +586,9 @@ class AnalysisServiceDownload(AnalysisService):
if needs_fresh_download:
logger.info("Performing request to TOAR DB")
+ if filename.is_file():#delete old status endpoint, in case a fresh download is requested for an successful request.
+ self.connection.cache.remove(query_options.cache_key)
response = self.connection.get(query_options)
- print(response.request.url)
with open(filename, "w+b") as downloaded_file:
downloaded_file.write(response.content)
else:
diff --git a/tests/conversionOfTimestamps.py b/tests/conversionOfTimestamps.py
index a14c4ba226d4ab15f45f88907e20200864191841..16e3a579b997beb1883198861d63c16717fa6ab6 100644
--- a/tests/conversionOfTimestamps.py
+++ b/tests/conversionOfTimestamps.py
@@ -56,7 +56,7 @@ analysis_service = AnalysisServiceDownload(
# maybe adopt the interval for requesting the results and the total duration, before the client pauses the requests.
# as the requests take about 45min, it is more suitable to wait 60min before timing out the requests than the original 30min.
-analysis_service.connection.setRequestTimes(interval_min=5, maxWait_min=60)
+analysis_service.connection.set_request_times(interval_min=5, max_wait_minutes=60)
createdFiles = []
diff --git a/tests/test_cache.py b/tests/test_cache.py
index 11e256c420e9c35065e1d01b604b9413d962c4a8..e856cd270aad36448d8386f583572825ea20edcd 100644
--- a/tests/test_cache.py
+++ b/tests/test_cache.py
@@ -15,7 +15,7 @@ def test_cache():
assert "test key 1" in myTestCache
assert "test key 2" in myTestCache
assert myTestCache.get("test key 1") == "test content 1"
- myTestCache.clearCache()
+ myTestCache.clear_cache()
assert "test key 2" in myTestCache
# edit creation date of key 2:
@@ -24,5 +24,5 @@ def test_cache():
assert "test key 2" in myTestCache
assert myTestCache.get("test key 2") == "other test content 2"
- myTestCache.clearCache()
+ myTestCache.clear_cache()
assert "test key 2" not in myTestCache