From c48c9f789114c62ef12d999c3881f5f0bb4aeeeb Mon Sep 17 00:00:00 2001
From: Matteo Westerwinter <m.westerwinter@fz-juelich.de>
Date: Tue, 18 Feb 2025 17:00:12 +0100
Subject: [PATCH] "/search/a?" is now handeled like "/search/?" refactored
aggregation into static method instead of using sum
---
tests/test_search_aggregations.py | 227 ++++++++++++++++++++++++++++++
toardb/timeseries/crud.py | 35 ++---
toardb/timeseries/timeseries.py | 13 +-
3 files changed, 255 insertions(+), 20 deletions(-)
diff --git a/tests/test_search_aggregations.py b/tests/test_search_aggregations.py
index 426d0a7..9ead2be 100644
--- a/tests/test_search_aggregations.py
+++ b/tests/test_search_aggregations.py
@@ -176,6 +176,233 @@ class TestApps:
)
db.execute("COMMIT")
+ def test_search_base(self, client, db):
+ response = client.get("/search/a?")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [
+ {
+ "id": 1,
+ "label": "CMA",
+ "order": 1,
+ "sampling_frequency": "hourly",
+ "aggregation": "mean",
+ "data_start_date": "2003-09-07T15:30:00+00:00",
+ "data_end_date": "2016-12-31T14:30:00+00:00",
+ "data_origin": "instrument",
+ "data_origin_type": "measurement",
+ "provider_version": "N/A",
+ "sampling_height": 7.0,
+ "additional_metadata": {},
+ "doi": "",
+ "coverage": -1.0,
+ "station": {
+ "id": 2,
+ "codes": ["SDZ54421"],
+ "name": "Shangdianzi",
+ "coordinates": {"lat": 40.65, "lng": 117.106, "alt": 293.9},
+ "coordinate_validation_status": "not checked",
+ "country": "China",
+ "state": "Beijing Shi",
+ "type": "unknown",
+ "type_of_area": "unknown",
+ "timezone": "Asia/Shanghai",
+ "additional_metadata": {"dummy_info": "Here is some more information about the station"},
+ "roles": [],
+ "annotations": [],
+ "aux_images": [],
+ "aux_docs": [],
+ "aux_urls": [],
+ "globalmeta": {
+ "mean_topography_srtm_alt_90m_year1994": -999.0,
+ "mean_topography_srtm_alt_1km_year1994": -999.0,
+ "max_topography_srtm_relative_alt_5km_year1994": -999.0,
+ "min_topography_srtm_relative_alt_5km_year1994": -999.0,
+ "stddev_topography_srtm_relative_alt_5km_year1994": -999.0,
+ "climatic_zone_year2016": "6 (warm temperate dry)",
+ "htap_region_tier1_year2010": "11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)",
+ "dominant_landcover_year2012": "11 (Cropland, rainfed, herbaceous cover)",
+ "landcover_description_25km_year2012": "",
+ "dominant_ecoregion_year2017": "-1 (undefined)",
+ "ecoregion_description_25km_year2017": "",
+ "distance_to_major_road_year2020": -999.0,
+ "mean_stable_nightlights_1km_year2013": -999.0,
+ "mean_stable_nightlights_5km_year2013": -999.0,
+ "max_stable_nightlights_25km_year2013": -999.0,
+ "max_stable_nightlights_25km_year1992": -999.0,
+ "mean_population_density_250m_year2015": -1.0,
+ "mean_population_density_5km_year2015": -1.0,
+ "max_population_density_25km_year2015": -1.0,
+ "mean_population_density_250m_year1990": -1.0,
+ "mean_population_density_5km_year1990": -1.0,
+ "max_population_density_25km_year1990": -1.0,
+ "mean_nox_emissions_10km_year2015": -999.0,
+ "mean_nox_emissions_10km_year2000": -999.0,
+ "toar1_category": "unclassified",
+ },
+ "changelog": [
+ {
+ "datetime": "2023-07-15T19:27:09.463245+00:00",
+ "description": "station created",
+ "old_value": "",
+ "new_value": "",
+ "station_id": 2,
+ "author_id": 1,
+ "type_of_change": "created",
+ }
+ ],
+ },
+ "variable": {
+ "name": "toluene",
+ "longname": "toluene",
+ "displayname": "Toluene",
+ "cf_standardname": "mole_fraction_of_toluene_in_air",
+ "units": "nmol mol-1",
+ "chemical_formula": "C7H8",
+ "id": 7,
+ },
+ "programme": {"id": 0, "name": "", "longname": "", "homepage": "", "description": ""},
+ "roles": [
+ {
+ "id": 2,
+ "role": "resource provider",
+ "status": "active",
+ "contact": {
+ "id": 4,
+ "organisation": {
+ "id": 1,
+ "name": "UBA",
+ "longname": "Umweltbundesamt",
+ "kind": "government",
+ "city": "Dessau-Roßlau",
+ "postcode": "06844",
+ "street_address": "Wörlitzer Platz 1",
+ "country": "Germany",
+ "homepage": "https://www.umweltbundesamt.de",
+ "contact_url": "mailto:immission@uba.de",
+ },
+ },
+ }
+ ],
+ },
+ {
+ "id": 2,
+ "label": "CMA",
+ "order": 1,
+ "sampling_frequency": "hourly",
+ "aggregation": "mean",
+ "data_start_date": "2003-09-07T15:30:00+00:00",
+ "data_end_date": "2016-12-31T14:30:00+00:00",
+ "data_origin": "instrument",
+ "data_origin_type": "measurement",
+ "provider_version": "N/A",
+ "sampling_height": 7.0,
+ "additional_metadata": {
+ "original_units": {"since_19740101000000": "nmol/mol"},
+ "measurement_method": "uv_abs",
+ "absorption_cross_section": "Hearn 1961",
+ "ebas_metadata_19740101000000_29y": {
+ "Submitter": "Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA",
+ "Data level": "2",
+ "Frameworks": "GAW-WDCRG NOAA-ESRL",
+ "Station code": "XXX",
+ "Station name": "Secret",
+ },
+ },
+ "doi": "",
+ "coverage": -1.0,
+ "station": {
+ "id": 3,
+ "codes": ["China_test8"],
+ "name": "Test_China",
+ "coordinates": {"lat": 36.256, "lng": 117.106, "alt": 1534.0},
+ "coordinate_validation_status": "not checked",
+ "country": "China",
+ "state": "Shandong Sheng",
+ "type": "unknown",
+ "type_of_area": "unknown",
+ "timezone": "Asia/Shanghai",
+ "additional_metadata": {},
+ "roles": [],
+ "annotations": [],
+ "aux_images": [],
+ "aux_docs": [],
+ "aux_urls": [],
+ "globalmeta": {
+ "mean_topography_srtm_alt_90m_year1994": -999.0,
+ "mean_topography_srtm_alt_1km_year1994": -999.0,
+ "max_topography_srtm_relative_alt_5km_year1994": -999.0,
+ "min_topography_srtm_relative_alt_5km_year1994": -999.0,
+ "stddev_topography_srtm_relative_alt_5km_year1994": -999.0,
+ "climatic_zone_year2016": "6 (warm temperate dry)",
+ "htap_region_tier1_year2010": "10 (SAF Sub Saharan/sub Sahel Africa)",
+ "dominant_landcover_year2012": "10 (Cropland, rainfed)",
+ "landcover_description_25km_year2012": "",
+ "dominant_ecoregion_year2017": "-1 (undefined)",
+ "ecoregion_description_25km_year2017": "",
+ "distance_to_major_road_year2020": -999.0,
+ "mean_stable_nightlights_1km_year2013": -999.0,
+ "mean_stable_nightlights_5km_year2013": -999.0,
+ "max_stable_nightlights_25km_year2013": -999.0,
+ "max_stable_nightlights_25km_year1992": -999.0,
+ "mean_population_density_250m_year2015": -1.0,
+ "mean_population_density_5km_year2015": -1.0,
+ "max_population_density_25km_year2015": -1.0,
+ "mean_population_density_250m_year1990": -1.0,
+ "mean_population_density_5km_year1990": -1.0,
+ "max_population_density_25km_year1990": -1.0,
+ "mean_nox_emissions_10km_year2015": -999.0,
+ "mean_nox_emissions_10km_year2000": -999.0,
+ "toar1_category": "unclassified",
+ },
+ "changelog": [
+ {
+ "datetime": "2023-08-15T21:16:20.596545+00:00",
+ "description": "station created",
+ "old_value": "",
+ "new_value": "",
+ "station_id": 3,
+ "author_id": 1,
+ "type_of_change": "created",
+ }
+ ],
+ },
+ "variable": {
+ "name": "o3",
+ "longname": "ozone",
+ "displayname": "Ozone",
+ "cf_standardname": "mole_fraction_of_ozone_in_air",
+ "units": "nmol mol-1",
+ "chemical_formula": "O3",
+ "id": 5,
+ },
+ "programme": {"id": 0, "name": "", "longname": "", "homepage": "", "description": ""},
+ "roles": [
+ {
+ "id": 1,
+ "role": "resource provider",
+ "status": "active",
+ "contact": {
+ "id": 5,
+ "organisation": {
+ "id": 2,
+ "name": "FZJ",
+ "longname": "Forschungszentrum Jülich",
+ "kind": "research",
+ "city": "Jülich",
+ "postcode": "52425",
+ "street_address": "Wilhelm-Johnen-Straße",
+ "country": "Germany",
+ "homepage": "https://www.fz-juelich.de",
+ "contact_url": "mailto:toar-data@fz-juelich.de",
+ },
+ },
+ }
+ ],
+ },
+ ]
+ assert response.json() == expected_resp
+
def test_search_single(self, client, db):
response = client.get("/search/a?id=2")
expected_status_code = 200
diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py
index f5a13a1..c1ad7c2 100644
--- a/toardb/timeseries/crud.py
+++ b/toardb/timeseries/crud.py
@@ -204,21 +204,24 @@ class TimeseriesQuery:
self.query = query
self.fields = fields
self.lconstr_roles = lconstr_roles
-
- def __add__(self, other):
- if self.fields != other.fields:
- raise ValueError("Fields of subquerys are diffrent")
-
- return TimeseriesQuery(
- True,
- self.query.union(other.query) if other.sign else self.query.except_(other.query),
- self.fields,
- self.lconstr_roles or other.lconstr_roles,
- )
-
- def __radd__(self, other):
- return self if other == 0 else self + other # To make sum() work
+ @staticmethod
+ def aggregate(querys):
+ aggregated_query = next(querys)
+ for query in querys:
+ if aggregated_query.fields != query.fields:
+ raise ValueError("Fields of subquerys are diffrent")
+
+ aggregated_query = TimeseriesQuery(
+ True,
+ aggregated_query.query.union(query.query)
+ if query.sign
+ else aggregated_query.query.except_(query.query),
+ aggregated_query.fields,
+ aggregated_query.lconstr_roles or query.lconstr_roles,
+ )
+ return aggregated_query
+
@staticmethod
def from_query_params(query_params, db, endpoint="timeseries", sign=True):
limit, offset, fields, format, filters = create_filter(query_params, endpoint)
@@ -303,11 +306,11 @@ def search_all_aggreagtion(db, path_params, signs, query_params_list, lts=False)
endpoint = "timeseries" if lts else "search"
try:
- return sum(
+ return TimeseriesQuery.aggregate(
TimeseriesQuery.from_query_params(query_params, db, endpoint, sign)
for sign, query_params in zip(signs, query_params_list)
).adapt_objects(db)
-
+
except (KeyError, ValueError) as e:
status_code = 400
return JSONResponse(status_code=status_code, content=str(e))
diff --git a/toardb/timeseries/timeseries.py b/toardb/timeseries/timeseries.py
index 445cefd..91df21b 100644
--- a/toardb/timeseries/timeseries.py
+++ b/toardb/timeseries/timeseries.py
@@ -50,10 +50,15 @@ def search_all_timeseries(request: Request, db: Session = Depends(get_db)):
@router.get('/search/a', response_model=List[schemas.Timeseries] | List[schemas.TimeseriesFields], response_model_exclude_none=True, response_model_exclude_unset=True)
def search_all_timeseries_aggregations(request: Request, db: Session = Depends(get_db)):
urls = re.split(r"(?=[+-]\D)", "+" + request.url.query)[1:]
-
- signs = [ url.startswith("+") for url in urls]
- query_params = [get_query_params(url[1:]) for url in urls]
- return crud.search_all_aggreagtion(db, path_params=request.path_params, signs=signs, query_params_list=query_params)
+ if urls:
+ signs = [url.startswith("+") for url in urls]
+ query_params = [get_query_params(url[1:]) for url in urls]
+ return crud.search_all_aggreagtion(
+ db, path_params=request.path_params, signs=signs, query_params_list=query_params
+ )
+ else:
+ updated_query_params = get_query_params(request.url.query)
+ return crud.search_all(db, path_params=request.path_params, query_params=updated_query_params)
--
GitLab