diff --git a/tests/fixtures/stationmeta/stationmeta_core.json b/tests/fixtures/stationmeta/stationmeta_core.json index 6120ac45aee7e3ff24c6f880edb94254c64c7ded..c2d1be388c000e49babfd2d2f1ae95626da1c37b 100644 --- a/tests/fixtures/stationmeta/stationmeta_core.json +++ b/tests/fixtures/stationmeta/stationmeta_core.json @@ -8,7 +8,7 @@ "type": 0, "type_of_area": 0, "timezone": 310, - "additional_metadata": {} + "additional_metadata": {"dummy_info": "Here is some more information about the station"} }, { "codes": ["SDZ54421"], @@ -19,7 +19,7 @@ "type": 0, "type_of_area": 0, "timezone": 310, - "additional_metadata": {"dummy_info": "Here is some more information about the station" } + "additional_metadata": {"add_type": "nature reservation"} }, { "codes":["China_test8"], diff --git a/tests/test_data.py b/tests/test_data.py index 2bf517a6a6809aa7583b4846a1811fb4ca6115de..b49a45623abb72268326abaa074baa2e51e8bddf 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -210,7 +210,7 @@ class TestApps: 'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi', 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9}, 'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'changelog': []}, 'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene', 'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1', 'chemical_formula': 'C7H8', 'id': 7}, @@ -285,8 +285,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some ' - 'more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], @@ -370,8 +369,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some ' - 'more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], @@ -458,7 +456,7 @@ class TestApps: '# "type_of_area": "unknown",\n', '# "timezone": "Asia/Shanghai",\n', '# "additional_metadata": {\n', - '# "dummy_info": "Here is some more information about the station"\n', + '# "add_type": "nature reservation"\n', '# },\n', '# "roles": null,\n', '# "annotations": null,\n', @@ -630,7 +628,7 @@ class TestApps: 'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi', 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9}, 'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'changelog': []}, 'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene', 'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1', 'chemical_formula': 'C7H8', 'id': 7}, @@ -1016,7 +1014,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], diff --git a/tests/test_search.py b/tests/test_search.py index 11636578c0d15db010b6a2a6fdc1a251f8df8a16..f1298459c496e5b10262e6c0478ff99489f2347a 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -200,7 +200,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], @@ -398,7 +398,8 @@ class TestApps: 'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Shandong Sheng', 'type': 'unknown', 'type_of_area': 'unknown', - 'timezone': 'Asia/Shanghai', 'additional_metadata': {}, + 'timezone': 'Asia/Shanghai', + 'additional_metadata': {}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)', 'distance_to_major_road_year2020': -999.0, @@ -916,3 +917,178 @@ class TestApps: }]}, 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}] assert response.json() == expected_resp + + + def test_search_with_additional_metadata(self, client, db): + response = client.get("/search/?additional_metadata->'absorption_cross_section'=Hearn1961") + expected_status_code = 200 + assert response.status_code == expected_status_code + expected_resp = [{'id': 2, 'label': 'CMA', 'order': 1, + 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement', + 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0, + 'data_origin': 'instrument', 'sampling_height': 7.0, + 'provider_version': 'N/A', + 'doi': '', + 'additional_metadata': {'absorption_cross_section': 'Hearn 1961', + 'measurement_method': 'uv_abs', + 'original_units': {'since_19740101000000': 'nmol/mol'}, + 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA', + 'Data level': '2', + 'Frameworks': 'GAW-WDCRG NOAA-ESRL', + 'Station code': 'XXX', + 'Station name': 'Secret' } }, + 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active', + 'contact': {'id': 5, 'organisation': {'id': 2, 'name': 'FZJ', 'longname': 'Forschungszentrum Jülich', + 'kind': 'research', 'city': 'Jülich', 'postcode': '52425', 'street_address': 'Wilhelm-Johnen-Straße', + 'country': 'Germany', 'homepage': 'https://www.fz-juelich.de', 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}], + 'variable': {'name': 'o3', 'longname': 'ozone', 'displayname': 'Ozone', + 'cf_standardname': 'mole_fraction_of_ozone_in_air', 'units': 'nmol mol-1', + 'chemical_formula': 'O3', 'id': 5}, + 'station': {'id': 3, 'codes': ['China_test8'], 'name': 'Test_China', + 'coordinates': {'alt': 1534.0, 'lat': 36.256, 'lng': 117.106}, + 'coordinate_validation_status': 'not checked', + 'country': 'China', 'state': 'Shandong Sheng', + 'type': 'unknown', 'type_of_area': 'unknown', + 'timezone': 'Asia/Shanghai', + 'additional_metadata': {}, + 'aux_images': [], 'aux_docs': [], 'aux_urls': [], + 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)', + 'distance_to_major_road_year2020': -999.0, + 'dominant_ecoregion_year2017': '-1 (undefined)', + 'dominant_landcover_year2012': '10 (Cropland, rainfed)', + 'ecoregion_description_25km_year2017': '', + 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)', + 'landcover_description_25km_year2012': '', + 'max_stable_nightlights_25km_year1992': -999.0, + 'max_stable_nightlights_25km_year2013': -999.0, + 'max_population_density_25km_year1990': -1.0, + 'max_population_density_25km_year2015': -1.0, + 'max_topography_srtm_relative_alt_5km_year1994': -999.0, + 'mean_stable_nightlights_1km_year2013': -999.0, + 'mean_stable_nightlights_5km_year2013': -999.0, + 'mean_nox_emissions_10km_year2000': -999.0, + 'mean_nox_emissions_10km_year2015': -999.0, + 'mean_population_density_250m_year1990': -1.0, + 'mean_population_density_250m_year2015': -1.0, + 'mean_population_density_5km_year1990': -1.0, + 'mean_population_density_5km_year2015': -1.0, + 'mean_topography_srtm_alt_1km_year1994': -999.0, + 'mean_topography_srtm_alt_90m_year1994': -999.0, + 'min_topography_srtm_relative_alt_5km_year1994': -999.0, + 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0, + 'toar1_category': 'unclassified'}, + 'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00', + 'description': 'station created', + 'old_value': '', + 'new_value': '', + 'station_id': 3, + 'author_id': 1, + 'type_of_change': 'created' + }]}, + 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}] + assert response.json() == expected_resp + + + def test_search_with_additional_metadata2(self, client, db): + response = client.get("/search/?additional_metadata->'absorption_cross_section'=Hearn1961"+ + "&additional_metadata->'sampling_type'=Continuous"+ + "&additional_metadata->'calibration_type'=Automatic") + expected_status_code = 200 + assert response.status_code == expected_status_code + expected_response = [] + assert response.json() == expected_response + + + def test_search_with_additional_metadata_unknown(self, client, db): + response = client.get("/search/?additional_metadata->'not_yet_defined'=42") + expected_status_code = 200 + assert response.status_code == expected_status_code + expected_response = [] + assert response.json() == expected_response + + + def test_search_with_additional_metadata_station(self, client, db): + response = client.get("/search/?station_additional_metadata->'add_type'=nature reservation") + expected_status_code = 200 + assert response.status_code == expected_status_code + expected_response = [{'id': 1, + 'label': 'CMA', + 'order': 1, + 'sampling_frequency': 'hourly', + 'aggregation': 'mean', + 'data_start_date': '2003-09-07T15:30:00+00:00', + 'data_end_date': '2016-12-31T14:30:00+00:00', + 'data_origin': 'instrument', + 'data_origin_type': 'measurement', + 'provider_version': 'N/A', + 'sampling_height': 7.0, + 'additional_metadata': {}, + 'doi': '', + 'coverage': -1.0, + 'station': {'id': 2, + 'codes': ['SDZ54421'], + 'name': 'Shangdianzi', + 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9}, + 'coordinate_validation_status': 'not checked', + 'country': 'China', + 'state': 'Beijing Shi', + 'type': 'unknown', + 'type_of_area': 'unknown', + 'timezone': 'Asia/Shanghai', + 'additional_metadata': {'add_type': 'nature reservation'}, + 'aux_images': [], + 'aux_docs': [], + 'aux_urls': [], + 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0, + 'mean_topography_srtm_alt_1km_year1994': -999.0, + 'max_topography_srtm_relative_alt_5km_year1994': -999.0, + 'min_topography_srtm_relative_alt_5km_year1994': -999.0, + 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0, + 'climatic_zone_year2016': '6 (warm temperate dry)', + 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)', + 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)', + 'landcover_description_25km_year2012': '', + 'dominant_ecoregion_year2017': '-1 (undefined)', + 'ecoregion_description_25km_year2017': '', + 'distance_to_major_road_year2020': -999.0, + 'mean_stable_nightlights_1km_year2013': -999.0, + 'mean_stable_nightlights_5km_year2013': -999.0, + 'max_stable_nightlights_25km_year2013': -999.0, + 'max_stable_nightlights_25km_year1992': -999.0, + 'mean_population_density_250m_year2015': -1.0, + 'mean_population_density_5km_year2015': -1.0, + 'max_population_density_25km_year2015': -1.0, + 'mean_population_density_250m_year1990': -1.0, + 'mean_population_density_5km_year1990': -1.0, + 'max_population_density_25km_year1990': -1.0, + 'mean_nox_emissions_10km_year2015': -999.0, + 'mean_nox_emissions_10km_year2000': -999.0, + 'toar1_category': 'unclassified'}, + 'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00', 'description': 'station created', 'old_value': '', 'new_value': '', 'station_id': 2, 'author_id': 1, 'type_of_change': 'created'}]}, + 'variable': {'name': 'toluene', + 'longname': 'toluene', + 'displayname': 'Toluene', + 'cf_standardname': 'mole_fraction_of_toluene_in_air', + 'units': 'nmol mol-1', + 'chemical_formula': 'C7H8', + 'id': 7}, + 'programme': {'id': 0, + 'name': '', + 'longname': '', + 'homepage': '', + 'description': ''}, + 'roles': [{'id': 2, + 'role': 'resource provider', + 'status': 'active', + 'contact': {'id': 4, + 'organisation': {'id': 1, + 'name': 'UBA', + 'longname': 'Umweltbundesamt', + 'kind': 'government', + 'city': 'Dessau-Roßlau', + 'postcode': '06844', + 'street_address': 'Wörlitzer Platz 1', + 'country': 'Germany', + 'homepage': 'https://www.umweltbundesamt.de', + 'contact_url': 'mailto:immission@uba.de'}}}]}] + assert response.json() == expected_response diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py index 42701fbcf9227879382ab00bf0250a519a96d867..0366dd3980d98552424f7dd66a015038fcac3fef 100644 --- a/tests/test_stationmeta.py +++ b/tests/test_stationmeta.py @@ -175,7 +175,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {}, + 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, 'roles': [{'id': 2, 'role': 'resource provider', 'status': 'active', @@ -259,8 +259,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more information about ' - 'the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active', @@ -599,6 +598,7 @@ class TestApps: 'id': 2, 'role': 'resource provider', 'status': 'active' }], + 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)', 'distance_to_major_road_year2020': -999.0, @@ -977,7 +977,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {}, + 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py index b2711a8dac1fa61be59d4db4917265e0fea65ccf..672d3b9e83ec0e01b8fe566579fd9a9762fe1722 100644 --- a/tests/test_timeseries.py +++ b/tests/test_timeseries.py @@ -202,9 +202,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more ' - 'information about the ' - 'station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], @@ -396,7 +394,7 @@ class TestApps: 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)', 'distance_to_major_road_year2020': -999.0, @@ -615,7 +613,7 @@ class TestApps: 'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)', 'distance_to_major_road_year2020': -999.0, @@ -711,7 +709,7 @@ class TestApps: 'type': 'unknown', 'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai', - 'additional_metadata': {'dummy_info': 'Here is some more information about the station'}, + 'additional_metadata': {'add_type': 'nature reservation'}, 'aux_images': [], 'aux_docs': [], 'aux_urls': [], diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py index 596b96a6228305f0f35baa08fadac13316d31316..cb732cd8aefa3d6bdca5e1847d47c2732ebd51b0 100644 --- a/toardb/timeseries/crud.py +++ b/toardb/timeseries/crud.py @@ -185,7 +185,6 @@ def search_all(db, path_params, query_params, lts=False): status_code=400 return JSONResponse(status_code=status_code, content=str(e)) lnot_role = (t_r_filter.find("NOT") > 0) - if fields: # sort input fields to be sure to replace station_changelog before changelog fields = ",".join(sorted(fields.split(','),reverse=True)) @@ -207,6 +206,8 @@ def search_all(db, path_params, query_params, lts=False): fields2.append("timeseries.order") elif field == "additional_metadata": fields2.append("timeseries.additional_metadata") + elif field == "station_additional_metadata": + fields2.append("stationmeta_core.additional_metadata") elif field == "station_id": fields2.append("stationmeta_core.id") elif field == "variable_id": diff --git a/toardb/utils/utils.py b/toardb/utils/utils.py index e84a4b2ee89aab6c8a64eb98fa12e980f318ce2f..d2da351446b21c74b68c475b6c675d928ba78506 100644 --- a/toardb/utils/utils.py +++ b/toardb/utils/utils.py @@ -144,15 +144,16 @@ def create_filter(query_params, endpoint): # determine allowed query parameters (first *only* from Timeseries) timeseries_params = {column.name for column in inspect(Timeseries).c} | {"has_role"} + timeseries_params = timeseries_params | {"additional_metadata-"} gis_params = {"bounding_box", "altitude_range"} if endpoint in ['search', 'timeseries']: core_params = {column.name for column in inspect(StationmetaCore).c if column.name not in ['id']} else: core_params = {column.name for column in inspect(StationmetaCore).c} - core_params |= {"globalmeta"} + core_params |= {"globalmeta", "station_additional_metadata-"} global_params = {column.name for column in inspect(StationmetaGlobal).c if column.name not in ['id','station_id']} data_params = {column.name for column in inspect(Data).c} | {"daterange", "format"} - ambig_params = {"station_id", "station_changelog", "station_country"} + ambig_params = {"station_id", "station_changelog", "station_country", "station_additional_metadata"} variable_params = {column.name for column in inspect(Variable).c} person_params = {column.name for column in inspect(Person).c} allrel_params = {"limit", "offset", "fields", "format"} @@ -205,12 +206,13 @@ def create_filter(query_params, endpoint): v_filter = [] p_filter = [] # query_params is a multi-dict! - for param in query_params: + for param_long in query_params: + param = param_long.split('>')[0] if param not in allowed_params: #inform user, that an unknown parameter name was used (this could be a typo and falsify the result!) raise KeyError(f"An unknown argument was received: {param}.") if param in allrel_params or param in profiling_params: continue - values = [item.strip() for v in query_params.getlist(param) for item in v.split(',')] + values = [item.strip() for v in query_params.getlist(param_long) for item in v.split(',')] # make sure ids are ints if param.endswith("id"): try: @@ -236,6 +238,8 @@ def create_filter(query_params, endpoint): values = translate_convoc_list(values, toardb.toardb.ST_vocabulary, "type") elif param == "type_of_area": values = translate_convoc_list(values, toardb.toardb.TA_vocabulary, "type of area") + elif param == "station_additional_metadata-": + param = f"{param_long[8:]}" # exceptions for special fields (codes, name) if param == 'codes': tmp_filter = [] @@ -245,6 +249,10 @@ def create_filter(query_params, endpoint): s_c_filter.append(f"({tmp_filter})") elif param == 'name': s_c_filter.append(f"LOWER(stationmeta_core.name) LIKE '%{values[0].lower()}%'") + elif param_long.split('>')[0] == "station_additional_metadata-": + val_mod = [ f"'\"{val}\"'::text" for val in values ] + values = ",".join(val_mod) + s_c_filter.append(f"to_json(stationmeta_core.{param})::text IN ({values})") else: s_c_filter.append(f"stationmeta_core.{param} IN {values}") elif endpoint in ["stationmeta", "search"] and param in global_params: @@ -276,6 +284,17 @@ def create_filter(query_params, endpoint): values = translate_convoc_list(values, toardb.toardb.OT_vocabulary, "data origin type") elif param == "data_origin": values = translate_convoc_list(values, toardb.toardb.DO_vocabulary, "data origin") + elif param == "additional_metadata-": + param = param_long + if param == "additional_metadata->'absorption_cross_section'": + trlist = translate_convoc_list(values, toardb.toardb.CS_vocabulary, "absorption_cross_section") + values = [ str(val) for val in trlist ] + if param == "additional_metadata->'sampling_type'": + trlist = translate_convoc_list(values, toardb.toardb.KS_vocabulary, "sampling_type") + values = [ str(val) for val in trlist ] + if param == "additional_metadata->'calibration_type'": + trlist = translate_convoc_list(values, toardb.toardb.CT_vocabulary, "calibration_type") + values = [ str(val) for val in trlist ] if param == "has_role": operator = "IN" join_operator = "OR"