From 6897c70fa4ea80f7e01f1598f235179214e6ce21 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Mon, 28 Oct 2024 13:38:43 +0000
Subject: [PATCH 01/36] #166: first profiling tools have been integrated using
middleware
---
toardb/toardb.py | 41 +++++++++++++++++++++++++++++++++++++++++
toardb/utils/utils.py | 4 +++-
2 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/toardb/toardb.py b/toardb/toardb.py
index eaf0973..962d504 100644
--- a/toardb/toardb.py
+++ b/toardb/toardb.py
@@ -22,6 +22,12 @@ from sqlalchemy.orm import Session
from starlette.responses import FileResponse
from fastapi.templating import Jinja2Templates
+from pyinstrument import Profiler
+from pyinstrument.renderers.html import HTMLRenderer
+from pyinstrument.renderers.speedscope import SpeedscopeRenderer
+from pathlib import Path
+import time
+
from toardb.utils.database import ToarDbSession, engine, get_db
from toardb.utils.settings import base_url
from toardb.utils.utils import normalize_metadata
@@ -61,6 +67,41 @@ app = FastAPI()
app.mount("/static", StaticFiles(directory="static"), name="_static")
templates = Jinja2Templates(directory="templates")
+
+# check more on https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-a-web-request-in-fastapi
+@app.middleware("http")
+async def profile_request(request: Request, call_next):
+ profile_type_to_ext = {"html": "html", "json": "json"}
+ profile_type_to_renderer = {
+ "html": HTMLRenderer,
+ "json": SpeedscopeRenderer,
+ }
+ if request.query_params.get("profile", False):
+ current_dir = Path(__file__).parent
+ profile_type = request.query_params.get("profile_format", "html")
+ with Profiler(interval=0.001, async_mode="enabled") as profiler:
+ response = await call_next(request)
+ ext = profile_type_to_ext[profile_type]
+ renderer = profile_type_to_renderer[profile_type]()
+ with open(current_dir / f"../profile.{ext}", "a") as outfile:
+ outfile.write(profiler.output(renderer=renderer))
+ return response
+ return await call_next(request)
+
+
+# check more on https://fastapi.tiangolo.com/tutorial/middleware/
+@app.middleware("http")
+async def add_process_time_header(request: Request, call_next):
+ if request.query_params.get("timing", False):
+ current_dir = Path(__file__).parent
+ start_time = time.time()
+ response = await call_next(request)
+ with open(current_dir / f"../timing.txt", "a") as outfile:
+ outfile.write("{} s: {}\n".format(time.time() - start_time, request.url))
+ return response
+ return await call_next(request)
+
+
@app.middleware("http")
async def response_to_csv(request: Request, call_next):
response = await call_next(request)
diff --git a/toardb/utils/utils.py b/toardb/utils/utils.py
index a30bc71..b3cae5b 100644
--- a/toardb/utils/utils.py
+++ b/toardb/utils/utils.py
@@ -150,6 +150,7 @@ def create_filter(query_params, endpoint):
variable_params = {column.name for column in inspect(Variable).c}
person_params = {column.name for column in inspect(Person).c}
allrel_params = {"limit", "offset", "fields", "format"}
+ profiling_params = {"profile", "profile_format", "timing"}
# pagination
offset= int(query_params.get("offset", 0))
@@ -169,6 +170,7 @@ def create_filter(query_params, endpoint):
format = query_params.get("format", 'json')
allowed_params = allrel_params.copy()
+ allowed_params |= profiling_params
if endpoint in {'stationmeta'}:
allowed_params |= gis_params | core_params | global_params
elif endpoint in {'timeseries'}:
@@ -176,7 +178,7 @@ def create_filter(query_params, endpoint):
elif endpoint in {'search'}:
allowed_params |= gis_params | core_params | global_params | timeseries_params | roles_params | ambig_params
elif endpoint in {'data'}:
- allowed_params = {"limit", "offset" } | data_params
+ allowed_params = {"limit", "offset" } | data_params | profiling_params
elif endpoint in {'variables'}:
allowed_params |= variable_params
elif endpoint in {'persons'}:
--
GitLab
From a75ab4d22996d070b3c5669019d043c113c2c90e Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Mon, 28 Oct 2024 14:07:06 +0000
Subject: [PATCH 02/36] #166: Since 'pyinstrument' is used for profiling, it
needs to be added to the 'requirements'.
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index 62b91ff..10ced33 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -33,6 +33,7 @@ postgis>=1.0.4
psycopg2-binary>=2.9.3
py>=1.11.0
pydantic==1.9.0
+pyinstrument
pytest==6.2.5
python-dateutil>=2.8.2
python-multipart>=0.0.5
--
GitLab
From 7e4e97bd598828f812b9b7909f8caf0b0d3b00fa Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Mon, 28 Oct 2024 15:41:58 +0000
Subject: [PATCH 03/36] #166: profiling arguments are not passed as filter
arguments.
---
toardb/utils/utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/toardb/utils/utils.py b/toardb/utils/utils.py
index b3cae5b..71316d8 100644
--- a/toardb/utils/utils.py
+++ b/toardb/utils/utils.py
@@ -204,7 +204,7 @@ def create_filter(query_params, endpoint):
for param in query_params:
if param not in allowed_params: #inform user, that an unknown parameter name was used (this could be a typo and falsify the result!)
raise KeyError(f"An unknown argument was received: {param}.")
- if param in allrel_params:
+ if param in allrel_params or param in profiling_params:
continue
values = [item.strip() for v in query_params.getlist(param) for item in v.split(',')]
# make sure ids are ints
--
GitLab
From 65e0a4ad6152721514683a3a2c8e2c8b1df2da30 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 22 Nov 2024 16:43:03 +0000
Subject: [PATCH 04/36] #170: method 'create_data_record_with_flag' has been
deleted
---
toardb/data/crud.py | 36 ------------------------------------
1 file changed, 36 deletions(-)
diff --git a/toardb/data/crud.py b/toardb/data/crud.py
index 6b878e2..d3d3aab 100644
--- a/toardb/data/crud.py
+++ b/toardb/data/crud.py
@@ -309,42 +309,6 @@ def create_data_record(db: Session, engine: Engine,
return JSONResponse(status_code=status_code, content=message)
-def create_data_record_with_flag(db: Session, engine: Engine,
- series_id: int, datetime: dt.datetime,
- value: float, flag: str, version: str,
- author_id: int):
- flag_num = get_value_from_str(toardb.toardb.DF_vocabulary,flag)
- data_dict = {"datetime": datetime,
- "value": value,
- "flags": flag_num,
- "version": version,
- "timeseries_id": series_id}
- data = models.Data(**data_dict)
- db.add(data)
- result = db.commit()
- db.refresh(data)
- # create changelog entry
- type_of_change = get_value_from_str(toardb.toardb.CL_vocabulary,"Created")
- description="data record created"
- db_changelog = TimeseriesChangelog(description=description, timeseries_id=series_id, author_id=author_id, type_of_change=type_of_change,
- old_value='', new_value='', period_start=datetime, period_end=datetime, version=version)
- db.add(db_changelog)
- db.commit()
- # adjust data_start_date, data_end_date
- timeseries = get_timeseries(db=db,timeseries_id=series_id)
- db.rollback()
- datetime = datetime.replace(tzinfo=timeseries.data_end_date.tzinfo)
- if datetime < timeseries.data_start_date:
- timeseries.data_start_date = datetime
- if datetime > timeseries.data_end_date:
- timeseries.data_end_date = datetime
- db.add(timeseries)
- db.commit()
- status_code=200
- message='Data successfully inserted.'
- return JSONResponse(status_code=status_code, content=message)
-
-
def insert_dataframe (db: Session, engine: Engine, df: pd.DataFrame, toarqc_config_type: str = 'standard', dry_run: bool = False, parameter: str = 'o3', preliminary: bool = False, force: bool = False):
# df: pandas.DataFrame
# index: datetime
--
GitLab
From 4fdba8a7064cc4ae291613ed5890ba3c17a788a2 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 22 Nov 2024 16:58:49 +0000
Subject: [PATCH 05/36] added more tests (especially for data flags)
---
tests/fixtures/data/data.json | 20 +++---
tests/test_data.py | 131 +++++++++++++++++++++++++++++-----
2 files changed, 125 insertions(+), 26 deletions(-)
diff --git a/tests/fixtures/data/data.json b/tests/fixtures/data/data.json
index 93b2db1..5b28000 100644
--- a/tests/fixtures/data/data.json
+++ b/tests/fixtures/data/data.json
@@ -23,14 +23,14 @@
{
"datetime":"2012-12-17 00:00:00+00",
"value":7.848,
- "flags":0,
+ "flags":1,
"timeseries_id":1,
"version":"000001.000000.00000000000000"
},
{
"datetime":"2012-12-17 01:00:00+00",
"value":15.696,
- "flags":0,
+ "flags":1,
"timeseries_id":1,
"version":"000001.000000.00000000000000"
},
@@ -44,21 +44,21 @@
{
"datetime":"2012-12-17 03:00:00+00",
"value":13.734,
- "flags":0,
+ "flags":2,
"timeseries_id":1,
"version":"000001.000000.00000000000000"
},
{
"datetime":"2012-12-17 04:00:00+00",
"value":19.62,
- "flags":0,
+ "flags":1,
"timeseries_id":1,
"version":"000001.000000.00000000000000"
},
{
"datetime":"2012-12-17 05:00:00+00",
"value":15.696,
- "flags":0,
+ "flags":2,
"timeseries_id":1,
"version":"000001.000000.00000000000000"
},
@@ -79,28 +79,28 @@
{
"datetime":"2012-12-16 22:00:00+00",
"value":13.734,
- "flags":0,
+ "flags":1,
"timeseries_id":2,
"version":"000001.000000.00000000000000"
},
{
"datetime":"2012-12-16 23:00:00+00",
"value":13.734,
- "flags":0,
+ "flags":1,
"timeseries_id":2,
"version":"000001.000000.00000000000000"
},
{
"datetime":"2012-12-17 00:00:00+00",
"value":7.848,
- "flags":0,
+ "flags":2,
"timeseries_id":2,
"version":"000001.000000.00000000000000"
},
{
"datetime":"2012-12-17 01:00:00+00",
"value":15.696,
- "flags":0,
+ "flags":2,
"timeseries_id":2,
"version":"000001.000000.00000000000000"
},
@@ -114,7 +114,7 @@
{
"datetime":"2012-12-17 03:00:00+00",
"value":13.734,
- "flags":0,
+ "flags":1,
"timeseries_id":2,
"version":"000001.000000.00000000000000"
},
diff --git a/tests/test_data.py b/tests/test_data.py
index d096718..7dd6f5a 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -202,7 +202,7 @@ class TestApps:
expected_resp = [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
{'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
- {'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'}]
+ {'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'}]
assert response.json() == expected_resp
@@ -239,15 +239,36 @@ class TestApps:
'data': [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
{'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
{'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
- {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
- {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
+ {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated QC passed', 'version': '1.0', 'timeseries_id': 1},
+ {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated QC passed', 'version': '1.0', 'timeseries_id': 1},
{'datetime': '2012-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
- {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
- {'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
- {'datetime': '2012-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
+ {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated modified', 'version': '1.0', 'timeseries_id': 1},
+ {'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated QC passed', 'version': '1.0', 'timeseries_id': 1},
+ {'datetime': '2012-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'OK validated modified', 'version': '1.0', 'timeseries_id': 1},
{'datetime': '2012-12-17T06:00:00+00:00', 'value': 5.886, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1}]
}
assert response.json() == expected_resp
+
+
+ # the data/map-endpoint is a special need of the analysis service
+ def test_get_map_data(self, client, db):
+ fixed_time = datetime(2023, 7, 28, 12, 0, 0)
+ with patch('toardb.timeseries.crud.dt.datetime') as mock_datetime:
+ mock_datetime.now.return_value = fixed_time
+ response = client.get("/data/map/?variable_id=7&daterange=2012-12-16T21:00,2012-12-17T06:00")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'timeseries_id': 1, 'value': 21.581},
+ {'timeseries_id': 1, 'value': 13.734},
+ {'timeseries_id': 1, 'value': 13.734},
+ {'timeseries_id': 1, 'value': 7.848},
+ {'timeseries_id': 1, 'value': 15.696},
+ {'timeseries_id': 1, 'value': 11.772},
+ {'timeseries_id': 1, 'value': 13.734},
+ {'timeseries_id': 1, 'value': 19.62},
+ {'timeseries_id': 1, 'value': 15.696},
+ {'timeseries_id': 1, 'value': 5.886}]
+ assert response.json() == expected_resp
# def test_insert_new_without_credits(self):
@@ -336,12 +357,12 @@ class TestApps:
'data': [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
{'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
{'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
- {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
- {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
+ {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated QC passed', 'timeseries_id': 1, 'version': '1.0'},
+ {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated QC passed', 'timeseries_id': 1, 'version': '1.0'},
{'datetime': '2012-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
- {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
- {'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
- {'datetime': '2012-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
+ {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated modified', 'timeseries_id': 1, 'version': '1.0'},
+ {'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated QC passed', 'timeseries_id': 1, 'version': '1.0'},
+ {'datetime': '2012-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'OK validated modified', 'timeseries_id': 1, 'version': '1.0'},
{'datetime': '2012-12-17T06:00:00+00:00', 'value': 5.886, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'}]}
assert response.json() == expected_response
@@ -432,11 +453,89 @@ class TestApps:
'citation': 'Forschungszentrum Jülich: time series of o3 at Test_China, accessed from the TOAR database on 2023-07-28 12:00:00',
'attribution': 'Test-Attributions to be announced',
'license': 'This data is published under a Creative Commons Attribution 4.0 International (CC BY 4.0). https://creativecommons.org/licenses/by/4.0/'},
- 'data': [{'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
- {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
- {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ 'data': [{'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated modified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated modified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'}]}
+ assert response.json() == expected_response
+
+
+ def test_get_data_with_specific_flags(self, client, db):
+ with patch('toardb.timeseries.crud.dt.datetime', FixedDatetime):
+ response = client.get("/data/timeseries/id/2?flags=OKValidatedVerified,OKValidatedQCPassed&daterange=2012-12-16%2023:00,%202012-12-17%2006:00")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = {'metadata': {'id': 2,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata': {'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'measurement_method': 'uv_abs',
+ 'absorption_cross_section': 0,
+ 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA', 'Data level': '2', 'Frameworks': 'GAW-WDCRG NOAA-ESRL', 'Station code': 'XXX', 'Station name': 'Secret'}},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 3,
+ 'codes': ['China_test8'],
+ 'name': 'Test_China',
+ 'coordinates': {'lat': 36.256, 'lng': 117.106, 'alt': 1534.0},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China',
+ 'state': 'Shandong Sheng',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': None,
+ 'changelog': []},
+ 'variable': {'name': 'o3',
+ 'longname': 'ozone',
+ 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3',
+ 'id': 5},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 1,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 5,
+ 'person': None,
+ 'organisation': {'id': 2,
+ 'name': 'FZJ',
+ 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research',
+ 'city': 'Jülich',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
+ 'changelog': None,
+ 'citation': 'Forschungszentrum Jülich: time series of o3 at Test_China, accessed from the TOAR database on 2023-07-28 12:00:00',
+ 'attribution': 'Test-Attributions to be announced',
+ 'license': 'This data is published under a Creative Commons Attribution 4.0 International (CC BY 4.0). https://creativecommons.org/licenses/by/4.0/'},
+ 'data': [{'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
{'datetime': '2012-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
- {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
{'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'}]}
assert response.json() == expected_response
@@ -555,7 +654,7 @@ class TestApps:
{'datetime': '2012-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'OK validated QC passed', 'timeseries_id': 1, 'version': '2.0'},
{'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 1, 'version': '2.0'},
{'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated QC passed', 'timeseries_id': 1, 'version': '2.0'},
- {'datetime': '2012-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
+ {'datetime': '2012-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'OK validated modified', 'timeseries_id': 1, 'version': '1.0'},
{'datetime': '2012-12-17T06:00:00+00:00', 'value': 5.886, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'}]
}
patched_response = response.json()
--
GitLab
From 60eed472c17946e1c36678c21b1a8112d11c360b Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Mon, 25 Nov 2024 13:50:46 +0000
Subject: [PATCH 06/36] added more tests for roles and globalmeta of stations
---
.../stationmeta_core_stationmeta_roles.json | 10 ++
.../stationmeta/stationmeta_roles.json | 14 ++
tests/test_stationmeta.py | 131 +++++++++++++++++-
toardb/stationmeta/crud.py | 3 +
4 files changed, 152 insertions(+), 6 deletions(-)
create mode 100644 tests/fixtures/stationmeta/stationmeta_core_stationmeta_roles.json
create mode 100644 tests/fixtures/stationmeta/stationmeta_roles.json
diff --git a/tests/fixtures/stationmeta/stationmeta_core_stationmeta_roles.json b/tests/fixtures/stationmeta/stationmeta_core_stationmeta_roles.json
new file mode 100644
index 0000000..e3b50af
--- /dev/null
+++ b/tests/fixtures/stationmeta/stationmeta_core_stationmeta_roles.json
@@ -0,0 +1,10 @@
+[
+ {
+ "station_id": 1,
+ "role_id": 2
+ },
+ {
+ "station_id": 2,
+ "role_id": 1
+ }
+]
diff --git a/tests/fixtures/stationmeta/stationmeta_roles.json b/tests/fixtures/stationmeta/stationmeta_roles.json
new file mode 100644
index 0000000..3d7dc6a
--- /dev/null
+++ b/tests/fixtures/stationmeta/stationmeta_roles.json
@@ -0,0 +1,14 @@
+[
+ {
+ "id": 1,
+ "role": 5,
+ "status": 0,
+ "contact_id": 5
+ },
+ {
+ "id": 2,
+ "role": 5,
+ "status": 0,
+ "contact_id": 4
+ }
+]
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index d8d7d95..0cde491 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -4,11 +4,14 @@
import pytest
import json
from fastapi import Request
+from sqlalchemy import insert
from toardb.stationmeta.models import (
StationmetaCore,
StationmetaGlobal,
StationmetaGlobalService,
- StationmetaChangelog
+ StationmetaRole,
+ StationmetaChangelog,
+ stationmeta_core_stationmeta_roles_table
)
from toardb.toardb import app
from toardb.stationmeta import crud
@@ -18,7 +21,7 @@ from toardb.stationmeta.schemas import (
StationmetaCreate
)
from toardb.auth_user.models import AuthUser
-from toardb.contacts.models import Person, Organisation
+from toardb.contacts.models import Person, Organisation, Contact
from toardb.test_base import (
client,
get_test_db,
@@ -75,7 +78,7 @@ class TestApps:
fake_conn.commit()
fake_cur.execute("ALTER SEQUENCE stationmeta_annotations_id_seq RESTART WITH 1")
fake_conn.commit()
- fake_cur.execute("ALTER SEQUENCE stationmeta_roles_id_seq RESTART WITH 1")
+ fake_cur.execute("ALTER SEQUENCE stationmeta_roles_id_seq RESTART WITH 3")
fake_conn.commit()
fake_cur.execute("ALTER SEQUENCE stationmeta_aux_doc_id_seq RESTART WITH 1")
fake_conn.commit()
@@ -87,6 +90,8 @@ class TestApps:
fake_conn.commit()
fake_cur.execute("ALTER SEQUENCE organisations_id_seq RESTART WITH 1")
fake_conn.commit()
+ fake_cur.execute("ALTER SEQUENCE contacts_id_seq RESTART WITH 1")
+ fake_conn.commit()
infilename = "tests/fixtures/auth_user/auth.json"
with open(infilename) as f:
metajson=json.load(f)
@@ -111,6 +116,14 @@ class TestApps:
db.add(new_organisation)
db.commit()
db.refresh(new_organisation)
+ infilename = "tests/fixtures/contacts/contacts.json"
+ with open(infilename) as f:
+ metajson=json.load(f)
+ for entry in metajson:
+ new_contact = Contact(**entry)
+ db.add(new_contact)
+ db.commit()
+ db.refresh(new_contact)
# I also need to upload tests with nested data!!!
infilename = "tests/fixtures/stationmeta/stationmeta_core.json"
with open(infilename) as f:
@@ -152,6 +165,20 @@ class TestApps:
db.add(new_stationmeta_global_service)
db.commit()
db.refresh(new_stationmeta_global_service)
+ infilename = "tests/fixtures/stationmeta/stationmeta_roles.json"
+ with open(infilename) as f:
+ metajson=json.load(f)
+ for entry in metajson:
+ new_stationmeta_role = StationmetaRole(**entry)
+ db.add(new_stationmeta_role)
+ db.commit()
+ db.refresh(new_stationmeta_role)
+ infilename = "tests/fixtures/stationmeta/stationmeta_core_stationmeta_roles.json"
+ with open(infilename) as f:
+ metajson=json.load(f)
+ for entry in metajson:
+ db.execute(insert(stationmeta_core_stationmeta_roles_table).values(station_id=entry["station_id"], role_id=entry["role_id"]))
+ db.execute("COMMIT")
# 1. tests retrieving station metadata
@@ -166,7 +193,22 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'roles': [{ 'contact': {
+ 'city': 'Dessau-Roßlau',
+ 'contact_url': 'mailto:immission@uba.de',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'id': 1,
+ 'kind': 'government',
+ 'longname': 'Umweltbundesamt',
+ 'name': 'UBA',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ },
+ 'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active' }],
+ 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -230,7 +272,22 @@ class TestApps:
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station' },
- 'roles': [], 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'roles': [{ 'contact': { 'city': 'Jülich',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'id': 2,
+ 'kind': 'research',
+ 'longname': 'Forschungszentrum Jülich',
+ 'name': 'FZJ',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße',
+ },
+ 'id': 1,
+ 'role': 'resource provider',
+ 'status': 'active' }],
+
+ 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -531,7 +588,22 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'roles': [{ 'contact': {
+ 'city': 'Dessau-Roßlau',
+ 'contact_url': 'mailto:immission@uba.de',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'id': 1,
+ 'kind': 'government',
+ 'longname': 'Umweltbundesamt',
+ 'name': 'UBA',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ },
+ 'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active' }],
+ 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -641,6 +713,26 @@ class TestApps:
assert response.json() == expected_resp
+ def test_insert_new_with_roles(self, client, db):
+ response = client.post("/stationmeta/",
+ json={"stationmeta":
+ {"codes":["ttt3","ttt4"],
+ "name":"Test_China","coordinates":{"lat":37.256,"lng":117.106,"alt":1534.0},
+ "coordinate_validation_status": "NotChecked",
+ "country":"CN","state":"Shandong Sheng",
+ "type":"Unknown","type_of_area":"Unknown","timezone":"Asia/Shanghai",
+ "roles": [{"role": "PointOfContact", "contact_id": 3, "status": "Active"},
+ {"role": "Originator", "contact_id": 1, "status": "Active"}],
+ "additional_metadata": "{}" }
+ }
+ )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': {'message': "new station: ['ttt3', 'ttt4'],Test_China,{'lat': 37.256, 'lng': 117.106, 'alt': 1534.0}",
+ 'station_id': 4}}
+ assert response.json() == expected_resp
+
+
def test_insert_new_same_coordinates(self, client, db):
response = client.post("/stationmeta/",
json={"stationmeta":
@@ -726,6 +818,33 @@ class TestApps:
assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
+ def test_patch_stationmeta_global(self, client, db):
+ response = client.patch("/stationmeta/SDZ54421?description=changing global metadata",
+ json={"stationmeta":
+ {"globalmeta": {"climatic_zone_year2016": "WarmTemperateMoist"}}
+# {"globalmeta": {"climatic_zone_year2016": "WarmTemperateMoist"},
+# "toar1_category": "RuralLowElevation",
+# "htap_region_tier1_year2010": "HTAPTier1PAN"}}
+ }
+ )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'message': 'patched stationmeta record for station_id 2', 'station_id': 2}
+ response_json = response.json()
+ assert response_json == expected_resp
+ response = client.get(f"/stationmeta/id/{response_json['station_id']}")
+ response_json = response.json()
+ # just check special changes
+ assert response_json['name'] == 'Shangdianzi'
+# assert response_json['changelog'][1]['old_value'] == "{'climatic_zone_year2016': 'WarmTemperateDry', 'toar1_category': 'RuralLowElevation', 'htap_region_tier1_year2010': 'HTAPTier1PAN'}"
+# assert response_json['changelog'][1]['new_value'] == "{'climatic_zone_year2016': 'WarmTemperateMoist', 'toar1_category': 'RuralLowElevation', 'htap_region_tier1_year2010': 'HTAPTier1PAN'}"
+ assert response_json['changelog'][1]['old_value'] == "{'climatic_zone_year2016': 'WarmTemperateDry'}"
+ assert response_json['changelog'][1]['new_value'] == "{'climatic_zone_year2016': 'WarmTemperateMoist'}"
+ assert response_json['changelog'][1]['author_id'] == 1
+# assert response_json['changelog'][1]['type_of_change'] == 'comprehensive metadata revision'
+ assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
+
+
def test_delete_roles_from_stationmeta(self, client, db):
response = client.patch("/stationmeta/delete_field/China11?field=roles")
expected_status_code = 200
diff --git a/toardb/stationmeta/crud.py b/toardb/stationmeta/crud.py
index 7425d30..237a06c 100644
--- a/toardb/stationmeta/crud.py
+++ b/toardb/stationmeta/crud.py
@@ -307,6 +307,9 @@ def create_stationmeta(db: Session, engine: Engine, stationmeta: StationmetaCrea
if db_object:
role_id = db_object.id
else:
+ # Something is going wrong here!
+ # Is the model StationmetaRole correctly defined?!
+ del db_role.contact
db.add(db_role)
db.commit()
db.refresh(db_role)
--
GitLab
From a7312dcc304443daf7e89fd946246b3e03fe634c Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 26 Nov 2024 09:29:06 +0000
Subject: [PATCH 07/36] correct units for global metadata coming from GeoPEAS
'nox_emissions'
---
templates/TOARDB_FASTAPI_Rest.html | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/templates/TOARDB_FASTAPI_Rest.html b/templates/TOARDB_FASTAPI_Rest.html
index 5fb39de..c1341c6 100644
--- a/templates/TOARDB_FASTAPI_Rest.html
+++ b/templates/TOARDB_FASTAPI_Rest.html
@@ -115,7 +115,7 @@
</div>
<div class="section" id="variables">
<h2>2.3 Variables<a class="headerlink" href="#variables" title="Permalink to this headline"></a></h2>
-<p><a class="reference external" href="https://toar-data-dev.fz-juelich.de/api/v2/variables/">https://toar-data-dev.fz-juelich.de/api/v2/variables/[name][?QUERY-OPTIONS]</a><br />or<br /><a class="reference external" href="https://toar-data-dev.fz-juelich.de/api/v3/variables/">https://toar-data-dev.fz-juelich.de/api/v2/variables/id/[id][?QUERY-OPTIONS]</a></p>
+<p><a class="reference external" href="https://toar-data-dev.fz-juelich.de/api/v2/variables/">https://toar-data-dev.fz-juelich.de/api/v2/variables/[name][?QUERY-OPTIONS]</a><br />or<br /><a class="reference external" href="https://toar-data-dev.fz-juelich.de/api/v2/variables/">https://toar-data-dev.fz-juelich.de/api/v2/variables/id/[id][?QUERY-OPTIONS]</a></p>
<p>where QUERY-OPTIONS are:</p>
<p>limit= <integer: count> (default: 10)<br />offset= <integer: number of elements to be skipped> (default: 0)</p>
<p>To get a list of all available variables use <em><strong>limit=None</strong></em>.</p>
@@ -317,12 +317,12 @@
<tr>
<td>mean_nox_emissions_10km_year2015</td>
<td>number</td>
-<td>mean value within a radius of 10 km around station location of the following data of the year 2015:<br>units: kg m-2 s-1<br>data_source: https://atmosphere.copernicus.eu/sites/default/files/2019-06/cams_emissions_general_document_apr2019_v7.pdf<br>citation: Granier, C., S. Darras, H. Denier van der Gon, J. Doubalova, N. Elguindi, B. Galle, M. Gauss, M. Guevara, J.-P. Jalkanen, J. Kuenen, C. Liousse, B. Quack, D. Simpson, K. Sindelarova The Copernicus Atmosphere Monitoring Service global and regional emissions (April 2019 version) Report April 2019 version null 2019 Elguindi, Granier, Stavrakou, Darras et al. Analysis of recent anthropogenic surface emissions from bottom-up inventories and top-down estimates: are future emission scenarios valid for the recent past? Earth's Future null submitted 2020</td>
+<td>mean value within a radius of 10 km around station location of the following data of the year 2015:<br>units: kg m-2 y-1<br>data_source: https://atmosphere.copernicus.eu/sites/default/files/2019-06/cams_emissions_general_document_apr2019_v7.pdf<br>citation: Granier, C., S. Darras, H. Denier van der Gon, J. Doubalova, N. Elguindi, B. Galle, M. Gauss, M. Guevara, J.-P. Jalkanen, J. Kuenen, C. Liousse, B. Quack, D. Simpson, K. Sindelarova The Copernicus Atmosphere Monitoring Service global and regional emissions (April 2019 version) Report April 2019 version null 2019 Elguindi, Granier, Stavrakou, Darras et al. Analysis of recent anthropogenic surface emissions from bottom-up inventories and top-down estimates: are future emission scenarios valid for the recent past? Earth's Future null submitted 2020</td>
</tr>
<tr>
<td>mean_nox_emissions_10km_year2000</td>
<td>number</td>
-<td>mean value within a radius of 10 km around station location of the following data of the year 2000:<br>units: kg m-2 s-1<br>data_source: https://atmosphere.copernicus.eu/sites/default/files/2019-06/cams_emissions_general_document_apr2019_v7.pdf<br>citation: Granier, C., S. Darras, H. Denier van der Gon, J. Doubalova, N. Elguindi, B. Galle, M. Gauss, M. Guevara, J.-P. Jalkanen, J. Kuenen, C. Liousse, B. Quack, D. Simpson, K. Sindelarova The Copernicus Atmosphere Monitoring Service global and regional emissions (April 2019 version) Report April 2019 version null 2019 Elguindi, Granier, Stavrakou, Darras et al. Analysis of recent anthropogenic surface emissions from bottom-up inventories and top-down estimates: are future emission scenarios valid for the recent past? Earth's Future null submitted 2020</td>
+<td>mean value within a radius of 10 km around station location of the following data of the year 2000:<br>units: kg m-2 y-1<br>data_source: https://atmosphere.copernicus.eu/sites/default/files/2019-06/cams_emissions_general_document_apr2019_v7.pdf<br>citation: Granier, C., S. Darras, H. Denier van der Gon, J. Doubalova, N. Elguindi, B. Galle, M. Gauss, M. Guevara, J.-P. Jalkanen, J. Kuenen, C. Liousse, B. Quack, D. Simpson, K. Sindelarova The Copernicus Atmosphere Monitoring Service global and regional emissions (April 2019 version) Report April 2019 version null 2019 Elguindi, Granier, Stavrakou, Darras et al. Analysis of recent anthropogenic surface emissions from bottom-up inventories and top-down estimates: are future emission scenarios valid for the recent past? Earth's Future null submitted 2020</td>
</tr>
</tbody>
</table>
--
GitLab
From a16f6b434dcd0a0b886c75e54f3936234a4bcec2 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 26 Nov 2024 14:33:22 +0000
Subject: [PATCH 08/36] made timeseries patching work with controlled
vocabulary; added more tests
---
tests/test_stationmeta.py | 49 +++++++++++++---
tests/test_timeseries.py | 99 ++++++++++++++++++++++++---------
toardb/timeseries/crud.py | 21 ++++++-
toardb/timeseries/timeseries.py | 5 +-
4 files changed, 138 insertions(+), 36 deletions(-)
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index 0cde491..834d039 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -818,13 +818,10 @@ class TestApps:
assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
- def test_patch_stationmeta_global(self, client, db):
+ def test_patch_single_stationmeta_global(self, client, db):
response = client.patch("/stationmeta/SDZ54421?description=changing global metadata",
json={"stationmeta":
{"globalmeta": {"climatic_zone_year2016": "WarmTemperateMoist"}}
-# {"globalmeta": {"climatic_zone_year2016": "WarmTemperateMoist"},
-# "toar1_category": "RuralLowElevation",
-# "htap_region_tier1_year2010": "HTAPTier1PAN"}}
}
)
expected_status_code = 200
@@ -836,12 +833,50 @@ class TestApps:
response_json = response.json()
# just check special changes
assert response_json['name'] == 'Shangdianzi'
-# assert response_json['changelog'][1]['old_value'] == "{'climatic_zone_year2016': 'WarmTemperateDry', 'toar1_category': 'RuralLowElevation', 'htap_region_tier1_year2010': 'HTAPTier1PAN'}"
-# assert response_json['changelog'][1]['new_value'] == "{'climatic_zone_year2016': 'WarmTemperateMoist', 'toar1_category': 'RuralLowElevation', 'htap_region_tier1_year2010': 'HTAPTier1PAN'}"
assert response_json['changelog'][1]['old_value'] == "{'climatic_zone_year2016': 'WarmTemperateDry'}"
assert response_json['changelog'][1]['new_value'] == "{'climatic_zone_year2016': 'WarmTemperateMoist'}"
assert response_json['changelog'][1]['author_id'] == 1
-# assert response_json['changelog'][1]['type_of_change'] == 'comprehensive metadata revision'
+ assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
+
+
+ def test_patch_multiple_stationmeta_global(self, client, db):
+ response = client.patch("/stationmeta/SDZ54421?description=changing global metadata",
+ json={"stationmeta":
+ {"globalmeta": {"climatic_zone_year2016": "WarmTemperateMoist",
+ "toar1_category": "RuralLowElevation",
+ "htap_region_tier1_year2010": "HTAPTier1PAN",
+ "dominant_landcover_year2012": "TreeNeedleleavedEvergreenClosedToOpen",
+ "landcover_description_25km_year2012": "TreeNeedleleavedEvergreenClosedToOpen: 100 %",
+ "dominant_ecoregion_year2017": "Guianansavanna",
+ "ecoregion_description_25km_year2017": "Guianansavanna: 90 %, Miskitopineforests: 10 %"}}
+ }
+ )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'message': 'patched stationmeta record for station_id 2', 'station_id': 2}
+ response_json = response.json()
+ assert response_json == expected_resp
+ response = client.get(f"/stationmeta/id/{response_json['station_id']}")
+ response_json = response.json()
+ # just check special changes
+ assert response_json['name'] == 'Shangdianzi'
+ assert response_json['changelog'][1]['old_value'] == (
+ "{'climatic_zone_year2016': 'WarmTemperateDry', "
+ "'htap_region_tier1_year2010': 'HTAPTier1SAF', "
+ "'dominant_landcover_year2012': 'CroplandRainfed', "
+ "'landcover_description_25km_year2012': '', "
+ "'dominant_ecoregion_year2017': 'Undefined', "
+ "'ecoregion_description_25km_year2017': '', "
+ "'toar1_category': 'Unclassified'}" )
+ assert response_json['changelog'][1]['new_value'] == (
+ "{'climatic_zone_year2016': 'WarmTemperateMoist', "
+ "'htap_region_tier1_year2010': 'HTAPTier1PAN', "
+ "'dominant_landcover_year2012': 'TreeNeedleleavedEvergreenClosedToOpen', "
+ "'landcover_description_25km_year2012': 'TreeNeedleleavedEvergreenClosedToOpen: 100 %', "
+ "'dominant_ecoregion_year2017': 'Guianansavanna', "
+ "'ecoregion_description_25km_year2017': 'Guianansavanna: 90 %, Miskitopineforests: 10 %'"
+ ", 'toar1_category': 'RuralLowElevation'}" )
+ assert response_json['changelog'][1]['author_id'] == 1
assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index f05cc43..21f7377 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -938,43 +938,88 @@ class TestApps:
expected_response = 'not a valid format: CMOR'
assert response.json() == expected_response
+ # 3. tests updating timeseries metadata
- """def test_get_timeseries_changelog(self, client, db):
- response = client.get("/timeseries_changelog/{id}")
- expected_status_code = 200
+ def test_patch_timeseries_no_description(self, client, db):
+ response = client.patch("/timeseries/id/1",
+ json={"timeseries":
+ {"sampling_frequency":"Daily"}
+ }
+ )
+ expected_status_code = 404
assert response.status_code == expected_status_code
- new_response = response.json()["datetime"] = ""
- expected_response = [{"datetime":"","description":"fake creation of timeseries","old_value":"","new_value":"","timeseries_id":8,"author_id":1,"type_of_change":0,"period_start":None,"period_end":None,"version":None}]
- assert response.json() == expected_response"""
+ expected_resp = {"detail": "description text ist missing."}
+ assert response.json() == expected_resp
- """ def test_patch_timeseries_error(self, client, db):
- response = client.patch("/timeseries/1",)
- expected_status_code = 400
+ def test_patch_timeseries_not_found(self, client, db):
+ response = client.patch("/timeseries/id/-1?description=changed sampling_frequency",
+ json={"timeseries":
+ {"sampling_frequency":"Daily"}
+ }
+ )
+ expected_status_code = 404
assert response.status_code == expected_status_code
- expected_response = {'detail': 'There was an error parsing the body'}
- assert response.json() == expected_response"""
-
-
- """def test_patch_timeseries(self, client, db):
- #print(client.get("/timeseries/1").json())
- response = client.patch("/timeseries/1?description=changed sampling_frequency",json={"timeseries": {"sampling_frequency": "daily"}})
- #print(client.get("/timeseries/1").json())
- assert False"""
+ expected_resp = {"detail": "Time series for patching not found."}
+ assert response.json() == expected_resp
- """def test_(self, client, db):
- response = client.get()
+ def test_patch_timeseries_single_item(self, client, db):
+ response = client.patch("/timeseries/id/1?description=changed sampling_frequency",
+ json={"timeseries":
+ {"sampling_frequency": "Daily"}
+ }
+ )
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_response = {}
- assert response.json() == expected_response
+ expected_resp = {'detail': { 'message': 'timeseries patched.',
+ 'timeseries_id': 1 }
+ }
+ response_json = response.json()
+ assert response_json == expected_resp
+ response = client.get(f"/timeseries/id/{response_json['detail']['timeseries_id']}")
+ response_json = response.json()
+ print(response_json)
+ # just check special changes
+ assert response_json['sampling_frequency'] == "daily"
+ assert response_json['changelog'][0]['old_value'] == "{'sampling_frequency': 'Hourly'}"
+ assert response_json['changelog'][0]['new_value'] == "{'sampling_frequency': 'Daily'}"
+ assert response_json['changelog'][0]['author_id'] == 1
+ assert response_json['changelog'][0]['type_of_change'] == 'single value correction in metadata'
+
+
+ def test_patch_timeseries_multiple_items(self, client, db):
+ response = client.patch("/timeseries/id/1?description=changed some metadata",
+ json={"timeseries":
+ {"sampling_frequency": "Daily",
+ "aggregation": "MeanOf4Samples",
+ "data_origin": "COSMOREA6",
+ "data_origin_type": "Model"}
+ }
+ )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': { 'message': 'timeseries patched.',
+ 'timeseries_id': 1 }
+ }
+ response_json = response.json()
+ assert response_json == expected_resp
+ response = client.get(f"/timeseries/id/{response_json['detail']['timeseries_id']}")
+ response_json = response.json()
+ print(response_json)
+ # just check special changes
+ assert response_json['sampling_frequency'] == "daily"
+ assert response_json['changelog'][0]['old_value'] == "{'sampling_frequency': 'Hourly', 'aggregation': 'Mean', 'data_origin': 'Instrument', 'data_origin_type': 'Measurement'}"
+ assert response_json['changelog'][0]['new_value'] == "{'sampling_frequency': 'Daily', 'aggregation': 'MeanOf4Samples', 'data_origin': 'COSMOREA6', 'data_origin_type': 'Model'}"
+ assert response_json['changelog'][0]['author_id'] == 1
+ assert response_json['changelog'][0]['type_of_change'] == 'comprehensive metadata revision'
- def test_(self, client, db):
- response = client.get()
+ """def test_get_timeseries_changelog(self, client, db):
+ response = client.get("/timeseries_changelog/{id}")
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_response = {}
- assert response.json() == expected_response
- """
+ new_response = response.json()["datetime"] = ""
+ expected_response = [{"datetime":"","description":"fake creation of timeseries","old_value":"","new_value":"","timeseries_id":8,"author_id":1,"type_of_change":0,"period_start":None,"period_end":None,"version":None}]
+ assert response.json() == expected_response"""
+
diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py
index db404f4..bc3a7ea 100644
--- a/toardb/timeseries/crud.py
+++ b/toardb/timeseries/crud.py
@@ -804,7 +804,16 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
if k == 'additional_metadata':
old_values[k] = db_timeseries.additional_metadata
else:
- old_values[k] = field
+ if k == "sampling_frequency":
+ old_values[k] = get_str_from_value(toardb.toardb.SF_vocabulary, int(field))
+ elif k == "aggregation":
+ old_values[k] = get_str_from_value(toardb.toardb.AT_vocabulary, int(field))
+ elif k == "data_origin":
+ old_values[k] = get_str_from_value(toardb.toardb.DO_vocabulary, int(field))
+ elif k == "data_origin_type":
+ old_values[k] = get_str_from_value(toardb.toardb.OT_vocabulary, int(field))
+ else:
+ old_values[k] = field
for k, v in timeseries_dict2.items():
setattr(db_timeseries,k,timeseries_dict[k])
# there is a mismatch with additional_metadata
@@ -812,6 +821,16 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
# but return from this method gives (=database): "additional_metadata": {}
# if timeseries_dict['additional_metadata']:
# db_timeseries.additional_metadata = clean_additional_metadata(db_timeseries.additional_metadata)
+ # do the following for every entry that uses the controlled vocabulary!
+ # this should be improved!
+ if db_obj.sampling_frequency:
+ db_timeseries.sampling_frequency = get_value_from_str(toardb.toardb.SF_vocabulary, db_obj.sampling_frequency)
+ if db_obj.aggregation:
+ db_timeseries.aggregation = get_value_from_str(toardb.toardb.AT_vocabulary, db_obj.aggregation)
+ if db_obj.data_origin:
+ db_timeseries.data_origin = get_value_from_str(toardb.toardb.DO_vocabulary, db_obj.data_origin)
+ if db_obj.data_origin_type:
+ db_timeseries.data_origin_type = get_value_from_str(toardb.toardb.OT_vocabulary, db_obj.data_origin_type)
db.add(db_timeseries)
result = db.commit()
# store roles and update association table
diff --git a/toardb/timeseries/timeseries.py b/toardb/timeseries/timeseries.py
index ae880cb..d5b5e21 100644
--- a/toardb/timeseries/timeseries.py
+++ b/toardb/timeseries/timeseries.py
@@ -133,12 +133,15 @@ def create_timeseries(timeseries: schemas.TimeseriesCreate = Body(..., embed = T
#@router.patch('/timeseries/id/{timeseries_id}', response_model=schemas.TimeseriesPatch)
@router.patch('/timeseries/id/{timeseries_id}')
def patch_timeseries(timeseries_id: int,
- description: str,
+ description: str = None,
timeseries: schemas.TimeseriesPatch = Body(..., embed = True),
access: dict = Depends(get_timeseries_md_change_access_rights),
db: Session = Depends(get_db)):
# check whether the patch is authorized (401: Unauthorized)
if access['status_code'] == 200:
+ # check, if description has been given in query_params!
+ if not description:
+ raise HTTPException(status_code=404, detail="description text ist missing.")
db_timeseries = crud.get_timeseries(db, timeseries_id=timeseries_id)
if db_timeseries is None:
raise HTTPException(status_code=404, detail="Time series for patching not found.")
--
GitLab
From 7a669219df93d80be62f7449ccf3d6a9bc0556ad Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Wed, 27 Nov 2024 14:47:09 +0000
Subject: [PATCH 09/36] added tests for data ingestion, version_numbers and
searches with global metadata filters
---
.../stationmeta/stationmeta_global.json | 4 +-
tests/test_data.py | 37 +++
tests/test_search.py | 307 +++++++++++++-----
tests/test_stationmeta.py | 286 ++++++++--------
tests/test_timeseries.py | 273 ++++++++++------
5 files changed, 582 insertions(+), 325 deletions(-)
diff --git a/tests/fixtures/stationmeta/stationmeta_global.json b/tests/fixtures/stationmeta/stationmeta_global.json
index 6e6be39..b084fe1 100644
--- a/tests/fixtures/stationmeta/stationmeta_global.json
+++ b/tests/fixtures/stationmeta/stationmeta_global.json
@@ -5,7 +5,7 @@
"min_topography_srtm_relative_alt_5km_year1994":-999.0,
"stddev_topography_srtm_relative_alt_5km_year1994":-999.0,
"climatic_zone_year2016": 6,
- "htap_region_tier1_year2010":10,
+ "htap_region_tier1_year2010":11,
"dominant_landcover_year2012":10,
"landcover_description_25km_year2012":"11:30.7 %,12:25.0 %,210:16.9 %,130:8.6 %,190:5.8 %,100:3.5 %,40:2.6 %,10:2.0 %,30:1.9 %",
"dominant_ecoregion_year2017":-1,
@@ -31,7 +31,7 @@
"min_topography_srtm_relative_alt_5km_year1994":-999.0,
"stddev_topography_srtm_relative_alt_5km_year1994":-999.0,
"climatic_zone_year2016": 6,
- "htap_region_tier1_year2010":10,
+ "htap_region_tier1_year2010":11,
"dominant_landcover_year2012":10,
"landcover_description_25km_year2012":"",
"dominant_ecoregion_year2017":-1,
diff --git a/tests/test_data.py b/tests/test_data.py
index 7dd6f5a..fe79617 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -539,6 +539,21 @@ class TestApps:
{'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'}]}
assert response.json() == expected_response
+ def test_create_data_record(self, client, db):
+ response = client.post("/data/timeseries/record/?series_id=2&datetime=2021-08-23%2015:00:00&value=67.3&flag=OK&version=000001.000001.00000000000000")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = 'Data successfully inserted.'
+ assert response.json() == expected_response
+ patched_response = client.get("/timeseries/2")
+ assert patched_response.status_code == expected_status_code
+ patched_response_changelog = patched_response.json()["changelog"][-1]
+ patched_response_changelog["datetime"] = ""
+ expected_changelog = {'datetime': '', 'description': 'data record created', 'old_value': '', 'new_value': '',
+ 'timeseries_id': 2, 'author_id': 1, 'type_of_change': 'created', 'period_start': '2021-08-23T15:00:00+00:00',
+ 'period_end': '2021-08-23T15:00:00+00:00', 'version': '000001.000001.00000000000000'}
+ assert patched_response_changelog == expected_changelog
+
def test_patch_data(self, client, db):
response = client.patch("/data/timeseries/?description=test patch&version=000002.000000.00000000000000", files={"file": open("tests/fixtures/data/toluene_SDZ54421_2012_2012_v2-0.dat", "rb")})
@@ -557,6 +572,17 @@ class TestApps:
def test_patch_bulk_data(self, client, db):
+ # show version of the original time series
+ version_response = client.get("/data/timeseries/next_version/1")
+ expected_status_code = 200
+ assert version_response.status_code == expected_status_code
+ expected_response = '000001.000001.00000000000000'
+ assert version_response.json() == expected_response
+ version_response = client.get("/data/timeseries/next_version/1?major=True")
+ expected_status_code = 200
+ assert version_response.status_code == expected_status_code
+ expected_response = '000002.000000.00000000000000'
+ assert version_response.json() == expected_response
response = client.patch("/data/timeseries/bulk/?description=test patch bulk data&version=000002.000000.00000000000000",
data='''[{"datetime": "2012-12-17T00:00:00+00:00", "value": 7.848, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"},
{"datetime": "2012-12-17T01:00:00+00:00", "value": 15.696, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"},
@@ -662,6 +688,17 @@ class TestApps:
dindex = patched_response['metadata']['citation'].index(" on 2")
patched_response['metadata']['citation'] = patched_response['metadata']['citation'][:dindex]
assert patched_response == expected_response
+ # also show that next_version has changed by this patch
+ version_response = client.get("/data/timeseries/next_version/1")
+ expected_status_code = 200
+ assert version_response.status_code == expected_status_code
+ expected_response = '000002.000001.00000000000000'
+ assert version_response.json() == expected_response
+ version_response = client.get("/data/timeseries/next_version/1?major=True")
+ expected_status_code = 200
+ assert version_response.status_code == expected_status_code
+ expected_response = '000003.000000.00000000000000'
+ assert version_response.json() == expected_response
def test_patch_bulk_data2(self, client, db):
diff --git a/tests/test_search.py b/tests/test_search.py
index b1e6ae4..fe1b96b 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -176,51 +176,60 @@ class TestApps:
response = client.get("/search/")
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_resp = [{'id': 1, 'label': 'CMA', 'order': 1,
- 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
- 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
- 'data_origin': 'instrument', 'sampling_height': 7.0,
- 'provider_version': 'N/A', 'doi': '', 'additional_metadata': {},
- 'roles': [{'id': 2, 'role': 'resource provider', 'status': 'active',
- 'contact': {'id': 4, 'organisation': {'id': 1, 'name': 'UBA', 'longname': 'Umweltbundesamt',
- 'kind': 'government', 'city': 'Dessau-Roßlau', 'postcode': '06844', 'street_address': 'Wörlitzer Platz 1',
- 'country': 'Germany', 'homepage': 'https://www.umweltbundesamt.de', 'contact_url': 'mailto:immission@uba.de'}}}],
- 'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
- 'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1',
- 'chemical_formula': 'C7H8', 'id': 7},
- 'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi',
+ expected_resp = [{'id': 1,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata': {},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 2,
+ 'codes': ['SDZ54421'],
+ 'name': 'Shangdianzi',
'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked',
- 'country': 'China', 'state': 'Beijing Shi',
- 'type': 'unknown', 'type_of_area': 'unknown',
+ 'country': 'China',
+ 'state': 'Beijing Shi',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'roles': [], 'annotations': [],
- 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
- 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
- 'distance_to_major_road_year2020': -999.0,
- 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
- 'ecoregion_description_25km_year2017': '',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
'landcover_description_25km_year2012': '',
- 'max_stable_nightlights_25km_year1992': -999.0,
- 'max_stable_nightlights_25km_year2013': -999.0,
- 'max_population_density_25km_year1990': -1.0,
- 'max_population_density_25km_year2015': -1.0,
- 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'ecoregion_description_25km_year2017': '',
+ 'distance_to_major_road_year2020': -999.0,
'mean_stable_nightlights_1km_year2013': -999.0,
'mean_stable_nightlights_5km_year2013': -999.0,
- 'mean_nox_emissions_10km_year2000': -999.0,
- 'mean_nox_emissions_10km_year2015': -999.0,
- 'mean_population_density_250m_year1990': -1.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
'mean_population_density_250m_year2015': -1.0,
- 'mean_population_density_5km_year1990': -1.0,
'mean_population_density_5km_year2015': -1.0,
- 'mean_topography_srtm_alt_1km_year1994': -999.0,
- 'mean_topography_srtm_alt_90m_year1994': -999.0,
- 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
'toar1_category': 'unclassified'},
'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00',
'description': 'station created',
@@ -228,62 +237,95 @@ class TestApps:
'new_value': '',
'station_id': 2,
'author_id': 1,
- 'type_of_change': 'created'
- }]},
- 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}},
- {'id': 2, 'label': 'CMA', 'order': 1,
- 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
- 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
- 'data_origin': 'instrument', 'sampling_height': 7.0,
+ 'type_of_change': 'created'}]},
+ 'variable': {'name': 'toluene',
+ 'longname': 'toluene',
+ 'displayname': 'Toluene',
+ 'cf_standardname': 'mole_fraction_of_toluene_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'C7H8',
+ 'id': 7},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 4,
+ 'organisation': {'id': 1,
+ 'name': 'UBA',
+ 'longname': 'Umweltbundesamt',
+ 'kind': 'government',
+ 'city': 'Dessau-Roßlau',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'contact_url': 'mailto:immission@uba.de'}}}]},
+ {'id': 2,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
'provider_version': 'N/A',
- 'doi': '',
- 'additional_metadata': {'absorption_cross_section': 'Hearn 1961',
+ 'sampling_height': 7.0,
+ 'additional_metadata': {'original_units': {'since_19740101000000': 'nmol/mol'},
'measurement_method': 'uv_abs',
- 'original_units': {'since_19740101000000': 'nmol/mol'},
- 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA',
- 'Data level': '2',
- 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
- 'Station code': 'XXX',
- 'Station name': 'Secret' } },
- 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active',
- 'contact': {'id': 5, 'organisation': {'id': 2, 'name': 'FZJ', 'longname': 'Forschungszentrum Jülich',
- 'kind': 'research', 'city': 'Jülich', 'postcode': '52425', 'street_address': 'Wilhelm-Johnen-Straße',
- 'country': 'Germany', 'homepage': 'https://www.fz-juelich.de', 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
- 'variable': {'name': 'o3', 'longname': 'ozone', 'displayname': 'Ozone',
- 'cf_standardname': 'mole_fraction_of_ozone_in_air', 'units': 'nmol mol-1',
- 'chemical_formula': 'O3', 'id': 5},
- 'station': {'id': 3, 'codes': ['China_test8'], 'name': 'Test_China',
- 'coordinates': {'alt': 1534.0, 'lat': 36.256, 'lng': 117.106},
+ 'absorption_cross_section': 'Hearn 1961',
+ 'ebas_metadata_19740101000000_29y':
+ {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret'}},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 3,
+ 'codes': ['China_test8'],
+ 'name': 'Test_China',
+ 'coordinates': {'lat': 36.256, 'lng': 117.106, 'alt': 1534.0},
'coordinate_validation_status': 'not checked',
- 'country': 'China', 'state': 'Shandong Sheng',
- 'type': 'unknown', 'type_of_area': 'unknown',
- 'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [],
- 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
- 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
- 'distance_to_major_road_year2020': -999.0,
- 'dominant_ecoregion_year2017': '-1 (undefined)',
- 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
- 'ecoregion_description_25km_year2017': '',
+ 'country': 'China',
+ 'state': 'Shandong Sheng',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'landcover_description_25km_year2012': '',
- 'max_stable_nightlights_25km_year1992': -999.0,
- 'max_stable_nightlights_25km_year2013': -999.0,
- 'max_population_density_25km_year1990': -1.0,
- 'max_population_density_25km_year2015': -1.0,
- 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'ecoregion_description_25km_year2017': '',
+ 'distance_to_major_road_year2020': -999.0,
'mean_stable_nightlights_1km_year2013': -999.0,
'mean_stable_nightlights_5km_year2013': -999.0,
- 'mean_nox_emissions_10km_year2000': -999.0,
- 'mean_nox_emissions_10km_year2015': -999.0,
- 'mean_population_density_250m_year1990': -1.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
'mean_population_density_250m_year2015': -1.0,
- 'mean_population_density_5km_year1990': -1.0,
'mean_population_density_5km_year2015': -1.0,
- 'mean_topography_srtm_alt_1km_year1994': -999.0,
- 'mean_topography_srtm_alt_90m_year1994': -999.0,
- 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
'toar1_category': 'unclassified'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
@@ -291,9 +333,34 @@ class TestApps:
'new_value': '',
'station_id': 3,
'author_id': 1,
- 'type_of_change': 'created'
- }]},
- 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
+ 'type_of_change': 'created'}]},
+ 'variable': {'name': 'o3',
+ 'longname': 'ozone',
+ 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3',
+ 'id': 5},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 1,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 5,
+ 'organisation':
+ {'id': 2,
+ 'name': 'FZJ',
+ 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research',
+ 'city': 'Jülich',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}]}]
assert response.json() == expected_resp
@@ -564,3 +631,73 @@ class TestApps:
}]},
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
assert response.json() == expected_resp
+
+
+ def test_search_with_global_attributes(self, client, db):
+ response = client.get("/search/?climatic_zone_year2016=WarmTemperateDry&htap_region_tier1_year2010=HTAPTier1SAF")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'id': 2, 'label': 'CMA', 'order': 1,
+ 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
+ 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
+ 'data_origin': 'instrument', 'sampling_height': 7.0,
+ 'provider_version': 'N/A',
+ 'doi': '',
+ 'additional_metadata': {'absorption_cross_section': 'Hearn 1961',
+ 'measurement_method': 'uv_abs',
+ 'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret' } },
+ 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active',
+ 'contact': {'id': 5, 'organisation': {'id': 2, 'name': 'FZJ', 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research', 'city': 'Jülich', 'postcode': '52425', 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany', 'homepage': 'https://www.fz-juelich.de', 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
+ 'variable': {'name': 'o3', 'longname': 'ozone', 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3', 'id': 5},
+ 'station': {'id': 3, 'codes': ['China_test8'], 'name': 'Test_China',
+ 'coordinates': {'alt': 1534.0, 'lat': 36.256, 'lng': 117.106},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China', 'state': 'Shandong Sheng',
+ 'type': 'unknown', 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai', 'additional_metadata': {},
+ 'roles': [], 'annotations': [],
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'distance_to_major_road_year2020': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'ecoregion_description_25km_year2017': '',
+ 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'landcover_description_25km_year2012': '',
+ 'max_stable_nightlights_25km_year1992': -999.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'mean_stable_nightlights_1km_year2013': -999.0,
+ 'mean_stable_nightlights_5km_year2013': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_250m_year2015': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'mean_population_density_5km_year2015': -1.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'toar1_category': 'unclassified'},
+ 'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
+ 'description': 'station created',
+ 'old_value': '',
+ 'new_value': '',
+ 'station_id': 3,
+ 'author_id': 1,
+ 'type_of_change': 'created'
+ }]},
+ 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
+ assert response.json() == expected_resp
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index 834d039..a3f6f59 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -187,75 +187,83 @@ class TestApps:
response = client.get("/stationmeta/")
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_resp = [{'id': 1, 'codes': ['China11'], 'name': 'Mount Tai',
+ expected_resp = [{'id': 1,
+ 'codes': ['China11'],
+ 'name': 'Mount Tai',
'coordinates': {'lat': 36.256, 'lng': 117.106, 'alt': 1534.0},
'coordinate_validation_status': 'not checked',
- 'country': 'China', 'state': 'Shandong Sheng',
- 'type': 'unknown', 'type_of_area': 'unknown',
- 'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [{ 'contact': {
- 'city': 'Dessau-Roßlau',
- 'contact_url': 'mailto:immission@uba.de',
- 'country': 'Germany',
- 'homepage': 'https://www.umweltbundesamt.de',
- 'id': 1,
- 'kind': 'government',
- 'longname': 'Umweltbundesamt',
- 'name': 'UBA',
- 'postcode': '06844',
- 'street_address': 'Wörlitzer Platz 1',
- },
- 'id': 2,
- 'role': 'resource provider',
- 'status': 'active' }],
- 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
- 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
- 'distance_to_major_road_year2020': -999.0,
- 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'country': 'China',
+ 'state': 'Shandong Sheng',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
+ 'roles': [{'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 1,
+ 'name': 'UBA',
+ 'longname': 'Umweltbundesamt',
+ 'kind': 'government',
+ 'city': 'Dessau-Roßlau',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'contact_url': 'mailto:immission@uba.de'}}],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. '
+ 'Arabia, Oman, etc, Iran, Iraq)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'landcover_description_25km_year2012': '11 (Cropland, '
+ 'rainfed, herbaceous '
+ 'cover): 30.7 %, 12 '
+ '(Cropland, rainfed, '
+ 'tree or shrub cover): '
+ '25.0 %, 210 (Water '
+ 'bodies): 16.9 %, 130 '
+ '(Grassland): 8.6 %, '
+ '190 (Urban areas): '
+ '5.8 %, 100 (Mosaic '
+ 'tree and shrub (>50%) '
+ '/ herbaceous cover '
+ '(<50%)): 3.5 %, 40 '
+ '(Mosaic natural '
+ 'vegetation (tree, '
+ 'shrub, herbaceous '
+ 'cover) (>50%) / '
+ 'cropland (<50%)): 2.6 '
+ '%, 10 (Cropland, '
+ 'rainfed): 2.0 %, 30 '
+ '(Mosaic cropland '
+ '(>50%) / natural '
+ 'vegetation (tree, '
+ 'shrub, herbaceous '
+ 'cover) (<50%)): 1.9 %',
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
'ecoregion_description_25km_year2017': '',
- 'landcover_description_25km_year2012': '11 (Cropland, rainfed, '
- +'herbaceous cover): '
- +'30.7 %, 12 (Cropland, '
- +'rainfed, tree or shrub '
- +'cover): 25.0 %, 210 '
- +'(Water bodies): 16.9 '
- +'%, 130 (Grassland): '
- +'8.6 %, 190 (Urban '
- +'areas): 5.8 %, 100 '
- +'(Mosaic tree and shrub '
- +'(>50%) / herbaceous '
- +'cover (<50%)): 3.5 %, '
- +'40 (Mosaic natural '
- +'vegetation (tree, '
- +'shrub, herbaceous '
- +'cover) (>50%) / '
- +'cropland (<50%)): 2.6 '
- +'%, 10 (Cropland, '
- +'rainfed): 2.0 %, 30 '
- +'(Mosaic cropland '
- +'(>50%) / natural '
- +'vegetation (tree, '
- +'shrub, herbaceous '
- +'cover) (<50%)): 1.9 %',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
- 'max_stable_nightlights_25km_year1992': -999.0,
- 'max_stable_nightlights_25km_year2013': -999.0,
- 'max_population_density_25km_year1990': -1.0,
- 'max_population_density_25km_year2015': -1.0,
- 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'distance_to_major_road_year2020': -999.0,
'mean_stable_nightlights_1km_year2013': -999.0,
'mean_stable_nightlights_5km_year2013': -999.0,
- 'mean_nox_emissions_10km_year2000': -999.0,
- 'mean_nox_emissions_10km_year2015': -999.0,
- 'mean_population_density_250m_year1990': -1.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
'mean_population_density_250m_year2015': -1.0,
- 'mean_population_density_5km_year1990': -1.0,
'mean_population_density_5km_year2015': -1.0,
- 'mean_topography_srtm_alt_1km_year1994': -999.0,
- 'mean_topography_srtm_alt_90m_year1994': -999.0,
- 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
'toar1_category': 'unclassified'},
'changelog': [{'datetime': '2023-07-05T08:23:04.551645+00:00',
'description': 'station created',
@@ -263,55 +271,61 @@ class TestApps:
'new_value': '',
'station_id': 1,
'author_id': 1,
- 'type_of_change': 'created'
- }]},
- {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi',
+ 'type_of_change': 'created'}]},
+ {'id': 2,
+ 'codes': ['SDZ54421'],
+ 'name': 'Shangdianzi',
'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked',
- 'country': 'China', 'state': 'Beijing Shi',
- 'type': 'unknown', 'type_of_area': 'unknown',
- 'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station' },
- 'roles': [{ 'contact': { 'city': 'Jülich',
- 'contact_url': 'mailto:toar-data@fz-juelich.de',
- 'country': 'Germany',
- 'homepage': 'https://www.fz-juelich.de',
- 'id': 2,
- 'kind': 'research',
- 'longname': 'Forschungszentrum Jülich',
- 'name': 'FZJ',
- 'postcode': '52425',
- 'street_address': 'Wilhelm-Johnen-Straße',
- },
- 'id': 1,
+ 'country': 'China',
+ 'state': 'Beijing Shi',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {'dummy_info': 'Here is some more information about '
+ 'the station'},
+ 'roles': [{'id': 1,
'role': 'resource provider',
- 'status': 'active' }],
-
- 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
- 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
- 'distance_to_major_road_year2020': -999.0,
- 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'status': 'active',
+ 'contact': {'id': 2,
+ 'name': 'FZJ',
+ 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research',
+ 'city': 'Jülich',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de'}}],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. '
+ 'Arabia, Oman, etc, Iran, Iraq)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'landcover_description_25km_year2012': '',
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
'ecoregion_description_25km_year2017': '',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
- 'max_stable_nightlights_25km_year1992': -999.0,
- 'max_stable_nightlights_25km_year2013': -999.0,
- 'max_population_density_25km_year1990': -1.0,
- 'max_population_density_25km_year2015': -1.0,
- 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'distance_to_major_road_year2020': -999.0,
'mean_stable_nightlights_1km_year2013': -999.0,
'mean_stable_nightlights_5km_year2013': -999.0,
- 'mean_nox_emissions_10km_year2000': -999.0,
- 'mean_nox_emissions_10km_year2015': -999.0,
- 'mean_population_density_250m_year1990': -1.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
'mean_population_density_250m_year2015': -1.0,
- 'mean_population_density_5km_year1990': -1.0,
'mean_population_density_5km_year2015': -1.0,
- 'mean_topography_srtm_alt_1km_year1994': -999.0,
- 'mean_topography_srtm_alt_90m_year1994': -999.0,
- 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
'toar1_category': 'unclassified'},
'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00',
'description': 'station created',
@@ -319,39 +333,48 @@ class TestApps:
'new_value': '',
'station_id': 2,
'author_id': 1,
- 'type_of_change': 'created'
- }]},
- {'id': 3, 'codes': ['China_test8'], 'name': 'Test_China',
+ 'type_of_change': 'created'}]},
+ {'id': 3,
+ 'codes': ['China_test8'],
+ 'name': 'Test_China',
'coordinates': {'lat': 36.256, 'lng': 117.106, 'alt': 1534.0},
'coordinate_validation_status': 'not checked',
- 'country': 'China', 'state': 'Shandong Sheng',
- 'type': 'unknown', 'type_of_area': 'unknown',
- 'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
- 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
- 'distance_to_major_road_year2020': -999.0,
- 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'country': 'China',
+ 'state': 'Shandong Sheng',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel '
+ 'Africa)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
- 'ecoregion_description_25km_year2017': '',
'landcover_description_25km_year2012': '',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
- 'max_stable_nightlights_25km_year1992': -999.0,
- 'max_stable_nightlights_25km_year2013': -999.0,
- 'max_population_density_25km_year1990': -1.0,
- 'max_population_density_25km_year2015': -1.0,
- 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'ecoregion_description_25km_year2017': '',
+ 'distance_to_major_road_year2020': -999.0,
'mean_stable_nightlights_1km_year2013': -999.0,
'mean_stable_nightlights_5km_year2013': -999.0,
- 'mean_nox_emissions_10km_year2000': -999.0,
- 'mean_nox_emissions_10km_year2015': -999.0,
- 'mean_population_density_250m_year1990': -1.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
'mean_population_density_250m_year2015': -1.0,
- 'mean_population_density_5km_year1990': -1.0,
'mean_population_density_5km_year2015': -1.0,
- 'mean_topography_srtm_alt_1km_year1994': -999.0,
- 'mean_topography_srtm_alt_90m_year1994': -999.0,
- 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
'toar1_category': 'unclassified'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
@@ -359,8 +382,7 @@ class TestApps:
'new_value': '',
'station_id': 3,
'author_id': 1,
- 'type_of_change': 'created'
- }]}]
+ 'type_of_change': 'created'}]}]
assert response.json() == expected_resp
@@ -424,7 +446,7 @@ class TestApps:
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
'climatic_zone_year2016': '6 (warm temperate dry)',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'landcover_description_25km_year2012': '11 (Cropland, rainfed, herbaceous cover): 30.7 %, 12 (Cropland, rainfed, tree or shrub cover): 25.0 %, 210 (Water bodies): 16.9 %, 130 (Grassland): 8.6 %, 190 (Urban areas): 5.8 %, 100 (Mosaic tree and shrub (>50%) / herbaceous cover (<50%)): 3.5 %, 40 (Mosaic natural vegetation (tree, shrub, herbaceous cover) (>50%) / cropland (<50%)): 2.6 %, 10 (Cropland, rainfed): 2.0 %, 30 (Mosaic cropland (>50%) / natural vegetation (tree, shrub, herbaceous cover) (<50%)): 1.9 %',
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -474,7 +496,7 @@ class TestApps:
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
'climatic_zone_year2016': '6 (warm temperate dry)',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'landcover_description_25km_year2012': '11 (Cropland, rainfed, herbaceous cover): 30.7 %, 12 (Cropland, rainfed, tree or shrub cover): 25.0 %, 210 (Water bodies): 16.9 %, 130 (Grassland): 8.6 %, 190 (Urban areas): 5.8 %, 100 (Mosaic tree and shrub (>50%) / herbaceous cover (<50%)): 3.5 %, 40 (Mosaic natural vegetation (tree, shrub, herbaceous cover) (>50%) / cropland (<50%)): 2.6 %, 10 (Cropland, rainfed): 2.0 %, 30 (Mosaic cropland (>50%) / natural vegetation (tree, shrub, herbaceous cover) (<50%)): 1.9 %',
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -502,7 +524,7 @@ class TestApps:
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
'climatic_zone_year2016': '6 (warm temperate dry)',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'landcover_description_25km_year2012': '',
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -633,7 +655,7 @@ class TestApps:
+'vegetation (tree, '
+'shrub, herbaceous '
+'cover) (<50%)): 1.9 %',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'max_stable_nightlights_25km_year1992': -999.0,
'max_stable_nightlights_25km_year2013': -999.0,
'max_population_density_25km_year1990': -1.0,
@@ -862,7 +884,7 @@ class TestApps:
assert response_json['name'] == 'Shangdianzi'
assert response_json['changelog'][1]['old_value'] == (
"{'climatic_zone_year2016': 'WarmTemperateDry', "
- "'htap_region_tier1_year2010': 'HTAPTier1SAF', "
+ "'htap_region_tier1_year2010': 'HTAPTier1MDE', "
"'dominant_landcover_year2012': 'CroplandRainfed', "
"'landcover_description_25km_year2012': '', "
"'dominant_ecoregion_year2017': 'Undefined', "
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index 21f7377..10745b7 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -187,140 +187,203 @@ class TestApps:
response = client.get("/timeseries/")
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_resp = [{'id': 1, 'label': 'CMA', 'order': 1,
- 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
- 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
- 'data_origin': 'instrument', 'sampling_height': 7.0,
- 'provider_version': 'N/A', 'additional_metadata': {},
+ expected_resp = [{'id': 1,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata': {},
'doi': '',
- 'roles': [{'id': 2, 'role': 'resource provider', 'status': 'active',
- 'contact': {'id': 4, 'organisation': {'id': 1, 'name': 'UBA', 'longname': 'Umweltbundesamt',
- 'kind': 'government', 'city': 'Dessau-Roßlau', 'postcode': '06844', 'street_address': 'Wörlitzer Platz 1',
- 'country': 'Germany', 'homepage': 'https://www.umweltbundesamt.de', 'contact_url': 'mailto:immission@uba.de'}}}],
- 'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
- 'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1',
- 'chemical_formula': 'C7H8', 'id': 7},
- 'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi',
+ 'coverage': -1.0,
+ 'station': {'id': 2,
+ 'codes': ['SDZ54421'],
+ 'name': 'Shangdianzi',
'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked',
- 'country': 'China', 'state': 'Beijing Shi',
- 'type': 'unknown', 'type_of_area': 'unknown',
+ 'country': 'China',
+ 'state': 'Beijing Shi',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'roles': [], 'annotations': [],
- 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
- 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
- 'distance_to_major_road_year2020': -999.0,
+ 'additional_metadata': {'dummy_info': 'Here is some more '
+ 'information about the '
+ 'station'},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle '
+ 'East: S. Arabia, '
+ 'Oman, etc, Iran, '
+ 'Iraq)',
+ 'dominant_landcover_year2012': '10 (Cropland, '
+ 'rainfed)',
+ 'landcover_description_25km_year2012': '',
'dominant_ecoregion_year2017': '-1 (undefined)',
- 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'ecoregion_description_25km_year2017': '',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
- 'landcover_description_25km_year2012': '',
- 'max_stable_nightlights_25km_year1992': -999.0,
- 'max_stable_nightlights_25km_year2013': -999.0,
- 'max_population_density_25km_year1990': -1.0,
- 'max_population_density_25km_year2015': -1.0,
- 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'distance_to_major_road_year2020': -999.0,
'mean_stable_nightlights_1km_year2013': -999.0,
'mean_stable_nightlights_5km_year2013': -999.0,
- 'mean_nox_emissions_10km_year2000': -999.0,
- 'mean_nox_emissions_10km_year2015': -999.0,
- 'mean_population_density_250m_year1990': -1.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
'mean_population_density_250m_year2015': -1.0,
- 'mean_population_density_5km_year1990': -1.0,
'mean_population_density_5km_year2015': -1.0,
- 'mean_topography_srtm_alt_1km_year1994': -999.0,
- 'mean_topography_srtm_alt_90m_year1994': -999.0,
- 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
'toar1_category': 'unclassified'},
'changelog': []},
- 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}},
+ 'variable': {'name': 'toluene',
+ 'longname': 'toluene',
+ 'displayname': 'Toluene',
+ 'cf_standardname': 'mole_fraction_of_toluene_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'C7H8',
+ 'id': 7},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 4,
+ 'organisation': {'id': 1,
+ 'name': 'UBA',
+ 'longname': 'Umweltbundesamt',
+ 'kind': 'government',
+ 'city': 'Dessau-Roßlau',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'contact_url': 'mailto:immission@uba.de'}}}]},
{'id': 2,
'label': 'CMA',
'order': 1,
- 'additional_metadata': {'absorption_cross_section': 'Hearn 1961',
- 'measurement_method': 'uv_abs',
- 'original_units': {'since_19740101000000': 'nmol/mol'},
- 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA',
- 'Data level': '2',
- 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
- 'Station code': 'XXX',
- 'Station name': 'Secret' } },
+ 'sampling_frequency': 'hourly',
'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
'data_end_date': '2016-12-31T14:30:00+00:00',
'data_origin': 'instrument',
'data_origin_type': 'measurement',
- 'data_start_date': '2003-09-07T15:30:00+00:00',
- 'coverage': -1.0,
- 'programme': {'description': '',
- 'homepage': '',
- 'id': 0,
- 'longname': '',
- 'name': ''},
'provider_version': 'N/A',
- 'doi': '',
- 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active',
- 'contact': {'id': 5, 'organisation': {'id': 2, 'name': 'FZJ', 'longname': 'Forschungszentrum Jülich',
- 'kind': 'research', 'city': 'Jülich', 'postcode': '52425', 'street_address': 'Wilhelm-Johnen-Straße',
- 'country': 'Germany', 'homepage': 'https://www.fz-juelich.de', 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
- 'sampling_frequency': 'hourly',
'sampling_height': 7.0,
- 'station': {'additional_metadata': {},
- 'annotations': [],
- 'aux_docs': [],
- 'aux_images': [],
- 'aux_urls': [],
- 'changelog': [],
+ 'additional_metadata': {'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'measurement_method': 'uv_abs',
+ 'absorption_cross_section': 'Hearn 1961',
+ 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, '
+ 'Lady, '
+ 'lady.unknown@unknown.com, '
+ 'some '
+ 'long '
+ 'division '
+ 'name, '
+ 'SHORT, '
+ ', '
+ '111 '
+ 'Streetname, '
+ ', '
+ 'zipcode, '
+ 'Boulder, '
+ 'CO, '
+ 'USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG '
+ 'NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret'}},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 3,
'codes': ['China_test8'],
+ 'name': 'Test_China',
+ 'coordinates': {'lat': 36.256, 'lng': 117.106, 'alt': 1534.0},
'coordinate_validation_status': 'not checked',
- 'coordinates': {'alt': 1534.0,
- 'lat': 36.256,
- 'lng': 117.106},
'country': 'China',
- 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
- 'distance_to_major_road_year2020': -999.0,
- 'dominant_ecoregion_year2017': '-1 (undefined)',
- 'dominant_landcover_year2012': '10 (Cropland, '
- 'rainfed)',
- 'ecoregion_description_25km_year2017': '',
+ 'state': 'Shandong Sheng',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
'htap_region_tier1_year2010': '10 (SAF Sub '
'Saharan/sub Sahel '
'Africa)',
+ 'dominant_landcover_year2012': '10 (Cropland, '
+ 'rainfed)',
'landcover_description_25km_year2012': '',
- 'max_population_density_25km_year1990': -1.0,
- 'max_population_density_25km_year2015': -1.0,
- 'max_stable_nightlights_25km_year1992': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'ecoregion_description_25km_year2017': '',
+ 'distance_to_major_road_year2020': -999.0,
+ 'mean_stable_nightlights_1km_year2013': -999.0,
+ 'mean_stable_nightlights_5km_year2013': -999.0,
'max_stable_nightlights_25km_year2013': -999.0,
- 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'mean_nox_emissions_10km_year2000': -999.0,
- 'mean_nox_emissions_10km_year2015': -999.0,
- 'mean_population_density_250m_year1990': -1.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
'mean_population_density_250m_year2015': -1.0,
- 'mean_population_density_5km_year1990': -1.0,
'mean_population_density_5km_year2015': -1.0,
- 'mean_stable_nightlights_1km_year2013': -999.0,
- 'mean_stable_nightlights_5km_year2013': -999.0,
- 'mean_topography_srtm_alt_1km_year1994': -999.0,
- 'mean_topography_srtm_alt_90m_year1994': -999.0,
- 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
'toar1_category': 'unclassified'},
- 'id': 3,
- 'name': 'Test_China',
- 'roles': [],
- 'state': 'Shandong Sheng',
- 'timezone': 'Asia/Shanghai',
- 'type': 'unknown',
- 'type_of_area': 'unknown'},
- 'variable': {'cf_standardname': 'mole_fraction_of_ozone_in_air',
- 'chemical_formula': 'O3',
- 'displayname': 'Ozone',
- 'id': 5,
+ 'changelog': []},
+ 'variable': {'name': 'o3',
'longname': 'ozone',
- 'name': 'o3',
- 'units': 'nmol mol-1'},
- }]
+ 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3',
+ 'id': 5},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 1,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 5,
+ 'organisation': {'id': 2,
+ 'name': 'FZJ',
+ 'longname': 'Forschungszentrum '
+ 'Jülich',
+ 'kind': 'research',
+ 'city': 'Jülich',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}]}]
assert response.json() == expected_resp
@@ -354,7 +417,7 @@ class TestApps:
'dominant_ecoregion_year2017': '-1 (undefined)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'ecoregion_description_25km_year2017': '',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'landcover_description_25km_year2012': '',
'max_stable_nightlights_25km_year1992': -999.0,
'max_stable_nightlights_25km_year2013': -999.0,
@@ -546,7 +609,7 @@ class TestApps:
'dominant_ecoregion_year2017': '-1 (undefined)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'ecoregion_description_25km_year2017': '',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'landcover_description_25km_year2012': '',
'max_stable_nightlights_25km_year1992': -999.0,
'max_stable_nightlights_25km_year2013': -999.0,
@@ -647,7 +710,7 @@ class TestApps:
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
'climatic_zone_year2016': '6 (warm temperate dry)',
- 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'dominant_landcover_year2012': '10 (Cropland, rainfed)',
'landcover_description_25km_year2012': '',
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -979,7 +1042,6 @@ class TestApps:
assert response_json == expected_resp
response = client.get(f"/timeseries/id/{response_json['detail']['timeseries_id']}")
response_json = response.json()
- print(response_json)
# just check special changes
assert response_json['sampling_frequency'] == "daily"
assert response_json['changelog'][0]['old_value'] == "{'sampling_frequency': 'Hourly'}"
@@ -1006,7 +1068,6 @@ class TestApps:
assert response_json == expected_resp
response = client.get(f"/timeseries/id/{response_json['detail']['timeseries_id']}")
response_json = response.json()
- print(response_json)
# just check special changes
assert response_json['sampling_frequency'] == "daily"
assert response_json['changelog'][0]['old_value'] == "{'sampling_frequency': 'Hourly', 'aggregation': 'Mean', 'data_origin': 'Instrument', 'data_origin_type': 'Measurement'}"
--
GitLab
From f986277209ebb6744ca5c92da99aab583b2d891f Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 29 Nov 2024 12:53:28 +0000
Subject: [PATCH 10/36] added tests for staging data
---
tests/fixtures/data/staging_data.json | 128 ++++++++++++++++++++++++++
tests/fixtures/toardb_pytest.psql | 18 ++++
tests/test_data.py | 117 +++++++++++++++++++++++
toardb/test_base.py | 1 +
4 files changed, 264 insertions(+)
create mode 100644 tests/fixtures/data/staging_data.json
diff --git a/tests/fixtures/data/staging_data.json b/tests/fixtures/data/staging_data.json
new file mode 100644
index 0000000..bf436b4
--- /dev/null
+++ b/tests/fixtures/data/staging_data.json
@@ -0,0 +1,128 @@
+[
+ {
+ "datetime":"2013-12-16 21:00:00+00",
+ "value":21.581,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-16 22:00:00+00",
+ "value":13.734,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-16 23:00:00+00",
+ "value":13.734,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 00:00:00+00",
+ "value":7.848,
+ "flags":11,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 01:00:00+00",
+ "value":15.696,
+ "flags":11,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 02:00:00+00",
+ "value":11.772,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 03:00:00+00",
+ "value":13.734,
+ "flags":12,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 04:00:00+00",
+ "value":19.62,
+ "flags":11,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 05:00:00+00",
+ "value":15.696,
+ "flags":12,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 06:00:00+00",
+ "value":5.886,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-16 21:00:00+00",
+ "value":21.581,
+ "flags":10,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-16 22:00:00+00",
+ "value":13.734,
+ "flags":11,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-16 23:00:00+00",
+ "value":13.734,
+ "flags":11,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 00:00:00+00",
+ "value":7.848,
+ "flags":12,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 01:00:00+00",
+ "value":15.696,
+ "flags":12,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 02:00:00+00",
+ "value":11.772,
+ "flags":10,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 03:00:00+00",
+ "value":13.734,
+ "flags":11,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 04:00:00+00",
+ "value":19.62,
+ "flags":10,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ }
+]
diff --git a/tests/fixtures/toardb_pytest.psql b/tests/fixtures/toardb_pytest.psql
index cad8262..372960d 100644
--- a/tests/fixtures/toardb_pytest.psql
+++ b/tests/fixtures/toardb_pytest.psql
@@ -2439,6 +2439,24 @@ CREATE TABLE IF NOT EXISTS public.data_archive (
ALTER TABLE public.data_archive OWNER TO postgres;
+CREATE SCHEMA staging;
+
+--
+-- Name: data; Type: TABLE; Schema: staging; Owner: postgres
+--
+
+CREATE TABLE IF NOT EXISTS staging.data (
+ datetime timestamp with time zone NOT NULL,
+ value double precision NOT NULL,
+ flags integer NOT NULL,
+ timeseries_id integer NOT NULL,
+ version character(28) DEFAULT '000001.000000.00000000000000'::bpchar NOT NULL,
+ CONSTRAINT data_archive_flags_check CHECK ((flags >= 0))
+);
+
+
+ALTER TABLE staging.data OWNER TO postgres;
+
--
-- Name: organisations; Type: TABLE; Schema: public; Owner: postgres
--
diff --git a/tests/test_data.py b/tests/test_data.py
index fe79617..8686a13 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -193,6 +193,15 @@ class TestApps:
db.add(new_data)
db.commit()
db.refresh(new_data)
+ infilename = "tests/fixtures/data/staging_data.json"
+ with open(infilename) as f:
+ metajson=json.load(f)
+ for entry in metajson:
+ new_data = Data (**entry)
+ fake_cur.execute(("INSERT INTO staging.data(datetime, value, flags, timeseries_id, version) "
+ f" VALUES('{new_data.datetime}', {new_data.value}, {new_data.flags}, "
+ f"{new_data.timeseries_id}, '{new_data.version}');"))
+ fake_conn.commit()
def test_get_data(self, client, db):
@@ -539,6 +548,114 @@ class TestApps:
{'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'}]}
assert response.json() == expected_response
+
+ def test_get_data_with_staging(self, client, db):
+ with patch('toardb.timeseries.crud.dt.datetime', FixedDatetime):
+ response = client.get("/data/timeseries_with_staging/id/2")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = {'metadata': {'id': 2,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata':
+ {'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'measurement_method': 'uv_abs',
+ 'absorption_cross_section': 0,
+ 'ebas_metadata_19740101000000_29y':
+ {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, '
+ 'some long division name, SHORT, , '
+ '111 Streetname, , zipcode, Boulder, CO, USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret'}},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 3,
+ 'codes': ['China_test8'],
+ 'name': 'Test_China',
+ 'coordinates': {'lat': 36.256, 'lng': 117.106, 'alt': 1534.0},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China',
+ 'state': 'Shandong Sheng',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': None,
+ 'changelog': []},
+ 'variable': {'name': 'o3',
+ 'longname': 'ozone',
+ 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3',
+ 'id': 5},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 1,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 5,
+ 'person': None,
+ 'organisation':
+ {'id': 2,
+ 'name': 'FZJ',
+ 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research',
+ 'city': 'Jülich',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
+ 'changelog': None,
+ 'citation': 'Forschungszentrum Jülich: time series of o3 at '
+ 'Test_China, accessed from the TOAR database on '
+ '2023-07-28 12:00:00',
+ 'attribution': 'Test-Attributions to be announced',
+ 'license': 'This data is published under a Creative Commons '
+ 'Attribution 4.0 International (CC BY 4.0). '
+ 'https://creativecommons.org/licenses/by/4.0/'
+ },
+ 'data': [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated modified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated modified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'questionable validated unconfirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'questionable validated unconfirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'questionable validated flagged', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'questionable validated unconfirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'questionable validated flagged', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T06:00:00+00:00', 'value': 5.886, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'}]
+ }
+ assert response.json() == expected_response
+
+
def test_create_data_record(self, client, db):
response = client.post("/data/timeseries/record/?series_id=2&datetime=2021-08-23%2015:00:00&value=67.3&flag=OK&version=000001.000001.00000000000000")
expected_status_code = 200
diff --git a/toardb/test_base.py b/toardb/test_base.py
index 0dd44eb..ad331ce 100644
--- a/toardb/test_base.py
+++ b/toardb/test_base.py
@@ -69,6 +69,7 @@ def test_db_session():
# otherwiese all tables from "toar_controlled_vocabulary" will get lost!
if not tbl.name.endswith("_vocabulary"):
_db_conn.execute(tbl.delete())
+ _db_conn.execute("DELETE FROM staging.data;")
fake_conn = _db_conn.raw_connection()
fake_cur = fake_conn.cursor()
fake_cur.execute("ALTER TABLE timeseries_changelog ALTER COLUMN datetime SET DEFAULT now();")
--
GitLab
From 40cf15c0d6153ee7a69f35971cf922c5c5e4f096 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 29 Nov 2024 15:15:42 +0000
Subject: [PATCH 11/36] add tests for patching time series roles and corrected
related changelog entry
---
tests/test_timeseries.py | 57 +++++++++++++++++++++++++++++++++++++++
toardb/timeseries/crud.py | 8 ++++++
2 files changed, 65 insertions(+)
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index 10745b7..63acb5f 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -1076,6 +1076,63 @@ class TestApps:
assert response_json['changelog'][0]['type_of_change'] == 'comprehensive metadata revision'
+ def test_patch_timeseries_roles(self, client, db):
+ response = client.patch("/timeseries/id/1?description=changed roles",
+ json={"timeseries":
+ {"roles": [{"role": "ResourceProvider", "contact_id": 5, "status": "Active"}]
+ }
+ }
+ )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': { 'message': 'timeseries patched.',
+ 'timeseries_id': 1 }
+ }
+ response_json = response.json()
+ assert response_json == expected_resp
+ response = client.get(f"/timeseries/id/{response_json['detail']['timeseries_id']}")
+ response_json = response.json()
+ # just check special changes
+ response_roles = [{'contact': {'id': 5,
+ 'organisation': {'city': 'Jülich',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'id': 2,
+ 'kind': 'research',
+ 'longname': 'Forschungszentrum Jülich',
+ 'name': 'FZJ',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße',
+ },
+ },
+ 'id': 1,
+ 'role': 'resource provider',
+ 'status': 'active'},
+ {'contact': {'id': 4,
+ 'organisation': {'city': 'Dessau-Roßlau',
+ 'contact_url': 'mailto:immission@uba.de',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'id': 1,
+ 'kind': 'government',
+ 'longname': 'Umweltbundesamt',
+ 'name': 'UBA',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ 'contact_url': 'mailto:immission@uba.de'}},
+ 'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active'}]
+ set_expected_response_roles = {json.dumps(item, sort_keys=True) for item in response_roles}
+ set_response_roles = {json.dumps(item, sort_keys=True) for item in response_json['roles']}
+ assert set_response_roles == set_expected_response_roles
+ assert response_json['changelog'][0]['old_value'] == "{'roles': [{'role': 'ResourceProvider', 'status': 'Active', 'contact_id': 4}]}"
+ assert response_json['changelog'][0]['new_value'] == "{'roles': [{'role': 'ResourceProvider', 'status': 'Active', 'contact_id': 4}, {'role': 'ResourceProvider', 'contact_id': 5, 'status': 'Active'}]}"
+ assert response_json['changelog'][0]['author_id'] == 1
+ assert response_json['changelog'][0]['type_of_change'] == 'single value correction in metadata'
+
+
"""def test_get_timeseries_changelog(self, client, db):
response = client.get("/timeseries_changelog/{id}")
expected_status_code = 200
diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py
index bc3a7ea..e700ef3 100644
--- a/toardb/timeseries/crud.py
+++ b/toardb/timeseries/crud.py
@@ -799,6 +799,7 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
no_log = (description == 'NOLOG')
if not no_log:
old_values={}
+ new_values={}
for k, v in timeseries_dict2.items():
field=str(getattr(db_timeseries,k))
if k == 'additional_metadata':
@@ -848,8 +849,11 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
old_value['contact_id'] = old_role.contact_id
old_roles.append(old_value)
old_values['roles'] = old_roles
+ new_roles = old_roles.copy()
for r in roles_data:
db_role = models.TimeseriesRole(**r)
+ if not no_log:
+ new_roles.append(r)
db_role.role = get_value_from_str(toardb.toardb.RC_vocabulary,db_role.role)
db_role.status = get_value_from_str(toardb.toardb.RS_vocabulary,db_role.status)
# check whether role is already present in database
@@ -863,6 +867,8 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
role_id = db_role.id
db.execute(insert(timeseries_timeseries_roles_table).values(timeseries_id=timeseries_id, role_id=role_id))
db.commit()
+ if not no_log:
+ new_values['roles'] = new_roles
# store annotations and update association table
if annotations_data:
if not no_log:
@@ -895,6 +901,8 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
db.commit()
# add patch to changelog table
if not no_log:
+ if new_values:
+ timeseries_dict2.update(new_values)
db_changelog = TimeseriesChangelog(description=description, timeseries_id=timeseries_id, author_id=author_id, type_of_change=type_of_change,
old_value=str(old_values), new_value=str(timeseries_dict2))
db.add(db_changelog)
--
GitLab
From 99db7a36bb8a712a92fd574b44467fe4c35c9591 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 29 Nov 2024 17:26:56 +0000
Subject: [PATCH 12/36] added test for stationmeta annotations
---
tests/test_stationmeta.py | 23 +++++++++++++++++++++++
toardb/stationmeta/crud.py | 1 +
toardb/stationmeta/schemas.py | 13 ++++++++++---
3 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index a3f6f59..5390f6b 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -755,6 +755,29 @@ class TestApps:
assert response.json() == expected_resp
+ def test_insert_new_with_annotations(self, client, db):
+ response = client.post("/stationmeta/",
+ json={"stationmeta":
+ {"codes":["ttt3","ttt4"],
+ "name":"Test_China","coordinates":{"lat":37.256,"lng":117.106,"alt":1534.0},
+ "coordinate_validation_status": "NotChecked",
+ "country":"CN","state":"Shandong Sheng",
+ "type":"Unknown","type_of_area":"Unknown","timezone":"Asia/Shanghai",
+ "annotations": [{"kind": "User",
+ "text": "some foo",
+ "date_added":"2021-07-27 00:00",
+ "approved": True,
+ "contributor_id":1}],
+ "additional_metadata": "{}" }
+ }
+ )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': {'message': "new station: ['ttt3', 'ttt4'],Test_China,{'lat': 37.256, 'lng': 117.106, 'alt': 1534.0}",
+ 'station_id': 4}}
+ assert response.json() == expected_resp
+
+
def test_insert_new_same_coordinates(self, client, db):
response = client.post("/stationmeta/",
json={"stationmeta":
diff --git a/toardb/stationmeta/crud.py b/toardb/stationmeta/crud.py
index 237a06c..2c1f869 100644
--- a/toardb/stationmeta/crud.py
+++ b/toardb/stationmeta/crud.py
@@ -320,6 +320,7 @@ def create_stationmeta(db: Session, engine: Engine, stationmeta: StationmetaCrea
if annotations_data:
for a in annotations_data:
db_annotation = models.StationmetaAnnotation(**a)
+ db_annotation.kind = get_value_from_str(toardb.toardb.AK_vocabulary,db_annotation.kind)
# check whether annotation is already present in database
db_object = get_unique_stationmeta_annotation(db, db_annotation.text, db_annotation.contributor_id)
if db_object:
diff --git a/toardb/stationmeta/schemas.py b/toardb/stationmeta/schemas.py
index a852efb..f544b44 100644
--- a/toardb/stationmeta/schemas.py
+++ b/toardb/stationmeta/schemas.py
@@ -142,7 +142,7 @@ def get_coordinates_from_string(point: str) -> Coordinates:
class StationmetaAnnotationBase(BaseModel):
id: int = None
- kind: int = Field(..., description="kind of annotation (see controlled vocabulary: Kind Of Annotation)")
+ kind: str = Field(..., description="kind of annotation (see controlled vocabulary: Kind Of Annotation)")
text: str = Field(..., description="text of annotation")
date_added: dt.datetime = Field(..., description="timestamp when annotation was added")
approved: bool = Field(..., description="Flag indicating whether the annotation of a station has been verified")
@@ -166,7 +166,14 @@ class StationmetaAnnotationPatch(BaseModel):
class StationmetaAnnotationCreate(StationmetaAnnotationBase):
- pass
+
+ @validator('kind')
+ def check_kind(cls, v):
+ if tuple(filter(lambda x: x.string == v, toardb.toardb.AK_vocabulary)):
+ return v
+ else:
+ raise ValueError(f"kind of annotation code not known: {v}")
+
class StationmetaAnnotation(StationmetaAnnotationBase):
@@ -696,7 +703,7 @@ class StationmetaPatch(StationmetaCorePatch):
class StationmetaCreate(StationmetaCoreCreate):
roles: List[StationmetaRoleCreate] = None
- annotations: List[StationmetaAnnotation] = None
+ annotations: List[StationmetaAnnotationCreate] = None
aux_images: List[StationmetaAuxImage] = None
aux_docs: List[StationmetaAuxDoc] = None
aux_urls: List[StationmetaAuxUrl] = None
--
GitLab
From 85fa86aeeff070a59bcef46ba5294411fe464b57 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Mon, 2 Dec 2024 11:01:16 +0000
Subject: [PATCH 13/36] added more tests for search filters
---
.../stationmeta/stationmeta_global.json | 2 +-
tests/test_search.py | 236 +++++++++++++++++-
tests/test_stationmeta.py | 6 +-
tests/test_timeseries.py | 10 +-
4 files changed, 240 insertions(+), 14 deletions(-)
diff --git a/tests/fixtures/stationmeta/stationmeta_global.json b/tests/fixtures/stationmeta/stationmeta_global.json
index b084fe1..d20512d 100644
--- a/tests/fixtures/stationmeta/stationmeta_global.json
+++ b/tests/fixtures/stationmeta/stationmeta_global.json
@@ -32,7 +32,7 @@
"stddev_topography_srtm_relative_alt_5km_year1994":-999.0,
"climatic_zone_year2016": 6,
"htap_region_tier1_year2010":11,
- "dominant_landcover_year2012":10,
+ "dominant_landcover_year2012":11,
"landcover_description_25km_year2012":"",
"dominant_ecoregion_year2017":-1,
"ecoregion_description_25km_year2017":"",
diff --git a/tests/test_search.py b/tests/test_search.py
index fe1b96b..5117b49 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -213,7 +213,7 @@ class TestApps:
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
'climatic_zone_year2016': '6 (warm temperate dry)',
'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
- 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)',
'landcover_description_25km_year2012': '',
'dominant_ecoregion_year2017': '-1 (undefined)',
'ecoregion_description_25km_year2017': '',
@@ -364,7 +364,15 @@ class TestApps:
assert response.json() == expected_resp
- def test_search_with_codes(self, client, db):
+ def test_search_variable_wrong_syntax(self, client, db):
+ response = client.get("/search/?variable_id=ozone")
+ expected_status_code = 400
+ assert response.status_code == expected_status_code
+ expected_resp = "Wrong value (not int) given: variable_id"
+ assert response.json() == expected_resp
+
+
+ def test_search_with_variable_id(self, client, db):
response = client.get("/search/?variable_id=5")
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -432,13 +440,161 @@ class TestApps:
}]},
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
assert response.json() == expected_resp
+
+
+ def test_search_with_codes(self, client, db):
+ response = client.get("/search/?codes=China_test8")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'id': 2, 'label': 'CMA', 'order': 1,
+ 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
+ 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
+ 'data_origin': 'instrument', 'sampling_height': 7.0,
+ 'provider_version': 'N/A',
+ 'doi': '',
+ 'additional_metadata': {'absorption_cross_section': 'Hearn 1961',
+ 'measurement_method': 'uv_abs',
+ 'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret' } },
+ 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active',
+ 'contact': {'id': 5, 'organisation': {'id': 2, 'name': 'FZJ', 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research', 'city': 'Jülich', 'postcode': '52425', 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany', 'homepage': 'https://www.fz-juelich.de', 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
+ 'variable': {'name': 'o3', 'longname': 'ozone', 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3', 'id': 5},
+ 'station': {'id': 3, 'codes': ['China_test8'], 'name': 'Test_China',
+ 'coordinates': {'alt': 1534.0, 'lat': 36.256, 'lng': 117.106},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China', 'state': 'Shandong Sheng',
+ 'type': 'unknown', 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai', 'additional_metadata': {},
+ 'roles': [], 'annotations': [],
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'distance_to_major_road_year2020': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'ecoregion_description_25km_year2017': '',
+ 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'landcover_description_25km_year2012': '',
+ 'max_stable_nightlights_25km_year1992': -999.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'mean_stable_nightlights_1km_year2013': -999.0,
+ 'mean_stable_nightlights_5km_year2013': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_250m_year2015': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'mean_population_density_5km_year2015': -1.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'toar1_category': 'unclassified'},
+ 'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
+ 'description': 'station created',
+ 'old_value': '',
+ 'new_value': '',
+ 'station_id': 3,
+ 'author_id': 1,
+ 'type_of_change': 'created'
+ }]},
+ 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
+ assert response.json() == expected_resp
- def test_search_wrong_syntax(self, client, db):
- response = client.get("/search/?variable_id=ozone")
+ def test_search_with_wrong_bounding_box(self, client, db):
+ response = client.get("/search/?bounding_box=Asia")
expected_status_code = 400
assert response.status_code == expected_status_code
- expected_resp = "Wrong value (not int) given: variable_id"
+ expected_resp = "not enough values to unpack (expected 4, got 1)"
+ assert response.json() == expected_resp
+
+
+ def test_search_empty_bounding_box(self, client, db):
+ response = client.get("/search/?bounding_box=117,36,118,37")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = []
+ assert response.json() == expected_resp
+
+
+ def test_search_with_bounding_box(self, client, db):
+ response = client.get("/search/?bounding_box=36,117,37,118")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'id': 2, 'label': 'CMA', 'order': 1,
+ 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
+ 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
+ 'data_origin': 'instrument', 'sampling_height': 7.0,
+ 'provider_version': 'N/A',
+ 'doi': '',
+ 'additional_metadata': {'absorption_cross_section': 'Hearn 1961',
+ 'measurement_method': 'uv_abs',
+ 'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret' } },
+ 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active',
+ 'contact': {'id': 5, 'organisation': {'id': 2, 'name': 'FZJ', 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research', 'city': 'Jülich', 'postcode': '52425', 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany', 'homepage': 'https://www.fz-juelich.de', 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
+ 'variable': {'name': 'o3', 'longname': 'ozone', 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3', 'id': 5},
+ 'station': {'id': 3, 'codes': ['China_test8'], 'name': 'Test_China',
+ 'coordinates': {'alt': 1534.0, 'lat': 36.256, 'lng': 117.106},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China', 'state': 'Shandong Sheng',
+ 'type': 'unknown', 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai', 'additional_metadata': {},
+ 'roles': [], 'annotations': [],
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'distance_to_major_road_year2020': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'ecoregion_description_25km_year2017': '',
+ 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'landcover_description_25km_year2012': '',
+ 'max_stable_nightlights_25km_year1992': -999.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'mean_stable_nightlights_1km_year2013': -999.0,
+ 'mean_stable_nightlights_5km_year2013': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_250m_year2015': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'mean_population_density_5km_year2015': -1.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'toar1_category': 'unclassified'},
+ 'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
+ 'description': 'station created',
+ 'old_value': '',
+ 'new_value': '',
+ 'station_id': 3,
+ 'author_id': 1,
+ 'type_of_change': 'created'
+ }]},
+ 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
assert response.json() == expected_resp
@@ -701,3 +857,73 @@ class TestApps:
}]},
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
assert response.json() == expected_resp
+
+
+ def test_search_with_global_attributes2(self, client, db):
+ response = client.get("/search/?dominant_landcover_year2012=CroplandRainfed&dominant_ecoregion_year2017=Undefined")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'id': 2, 'label': 'CMA', 'order': 1,
+ 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
+ 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
+ 'data_origin': 'instrument', 'sampling_height': 7.0,
+ 'provider_version': 'N/A',
+ 'doi': '',
+ 'additional_metadata': {'absorption_cross_section': 'Hearn 1961',
+ 'measurement_method': 'uv_abs',
+ 'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret' } },
+ 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active',
+ 'contact': {'id': 5, 'organisation': {'id': 2, 'name': 'FZJ', 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research', 'city': 'Jülich', 'postcode': '52425', 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany', 'homepage': 'https://www.fz-juelich.de', 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
+ 'variable': {'name': 'o3', 'longname': 'ozone', 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3', 'id': 5},
+ 'station': {'id': 3, 'codes': ['China_test8'], 'name': 'Test_China',
+ 'coordinates': {'alt': 1534.0, 'lat': 36.256, 'lng': 117.106},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China', 'state': 'Shandong Sheng',
+ 'type': 'unknown', 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai', 'additional_metadata': {},
+ 'roles': [], 'annotations': [],
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'distance_to_major_road_year2020': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'ecoregion_description_25km_year2017': '',
+ 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'landcover_description_25km_year2012': '',
+ 'max_stable_nightlights_25km_year1992': -999.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'mean_stable_nightlights_1km_year2013': -999.0,
+ 'mean_stable_nightlights_5km_year2013': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_250m_year2015': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'mean_population_density_5km_year2015': -1.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'toar1_category': 'unclassified'},
+ 'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
+ 'description': 'station created',
+ 'old_value': '',
+ 'new_value': '',
+ 'station_id': 3,
+ 'author_id': 1,
+ 'type_of_change': 'created'
+ }]},
+ 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
+ assert response.json() == expected_resp
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index 5390f6b..608ce9a 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -309,7 +309,7 @@ class TestApps:
'climatic_zone_year2016': '6 (warm temperate dry)',
'htap_region_tier1_year2010': '11 (MDE Middle East: S. '
'Arabia, Oman, etc, Iran, Iraq)',
- 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)',
'landcover_description_25km_year2012': '',
'dominant_ecoregion_year2017': '-1 (undefined)',
'ecoregion_description_25km_year2017': '',
@@ -525,7 +525,7 @@ class TestApps:
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
'climatic_zone_year2016': '6 (warm temperate dry)',
'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
- 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)',
'landcover_description_25km_year2012': '',
'dominant_ecoregion_year2017': '-1 (undefined)',
'ecoregion_description_25km_year2017': '',
@@ -908,7 +908,7 @@ class TestApps:
assert response_json['changelog'][1]['old_value'] == (
"{'climatic_zone_year2016': 'WarmTemperateDry', "
"'htap_region_tier1_year2010': 'HTAPTier1MDE', "
- "'dominant_landcover_year2012': 'CroplandRainfed', "
+ "'dominant_landcover_year2012': 'CroplandRainfedHerbaceousCover', "
"'landcover_description_25km_year2012': '', "
"'dominant_ecoregion_year2017': 'Undefined', "
"'ecoregion_description_25km_year2017': '', "
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index 63acb5f..6dba596 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -229,8 +229,8 @@ class TestApps:
'East: S. Arabia, '
'Oman, etc, Iran, '
'Iraq)',
- 'dominant_landcover_year2012': '10 (Cropland, '
- 'rainfed)',
+ 'dominant_landcover_year2012': '11 (Cropland, '
+ 'rainfed, herbaceous cover)',
'landcover_description_25km_year2012': '',
'dominant_ecoregion_year2017': '-1 (undefined)',
'ecoregion_description_25km_year2017': '',
@@ -415,7 +415,7 @@ class TestApps:
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
'dominant_ecoregion_year2017': '-1 (undefined)',
- 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)',
'ecoregion_description_25km_year2017': '',
'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'landcover_description_25km_year2012': '',
@@ -607,7 +607,7 @@ class TestApps:
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
'dominant_ecoregion_year2017': '-1 (undefined)',
- 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)',
'ecoregion_description_25km_year2017': '',
'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
'landcover_description_25km_year2012': '',
@@ -711,7 +711,7 @@ class TestApps:
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
'climatic_zone_year2016': '6 (warm temperate dry)',
'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
- 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)',
'landcover_description_25km_year2012': '',
'dominant_ecoregion_year2017': '-1 (undefined)',
'ecoregion_description_25km_year2017': '',
--
GitLab
From ecad26b6994bd493108c3d077cec354851fb1b52 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 3 Dec 2024 15:41:01 +0000
Subject: [PATCH 14/36] patching time series annotations works now; added more
tests for accessing data
---
tests/test_data.py | 36 +++++++++++++++++++++++++++++++++++-
tests/test_timeseries.py | 33 +++++++++++++++++++++++++++++++++
toardb/data/data.py | 4 ++--
toardb/timeseries/crud.py | 18 +++++++++++++++++-
toardb/timeseries/schemas.py | 34 +++++++++++++++++++++++++++++-----
5 files changed, 116 insertions(+), 9 deletions(-)
diff --git a/tests/test_data.py b/tests/test_data.py
index 8686a13..565f073 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -3,6 +3,7 @@
import pytest
import json
+import pandas as pd
from sqlalchemy import insert
from fastapi import Request
from toardb.toardb import app
@@ -46,7 +47,7 @@ async def override_dependency(request: Request):
if db_user:
auth_user_id = db_user.id
else: # pragma: no cover
- # the user needs to be add to the database!
+ # the user needs to be added to the database!
pass
access_dict = { "status_code": 200,
"user_name": "Sabine Schröder",
@@ -280,6 +281,17 @@ class TestApps:
assert response.json() == expected_resp
+ def test_get_no_data_with_variable_and_timerage(self, client, db):
+ # see: https://gitlab.jsc.fz-juelich.de/esde/toar-data/toardb_fastapi/-/issues/171
+ response = client.get("/data/map/?variable_id=25&daterange=2012-12-16T21:00,2012-12-17T06:00")
+# expected_status_code = 404
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+# expected_response = {'detail': 'Data not found.'}
+ expected_response = []
+ assert response.json() == expected_response
+
+
# def test_insert_new_without_credits(self):
#? response = client.post("/data/timeseries/")
# expected_status_code=401
@@ -311,6 +323,20 @@ class TestApps:
assert response.status_code == expected_status_code
expected_resp = {'detail': 'Data for timeseries already registered.'}
assert response.json() == expected_resp
+
+
+ def test_insert_new_as_bulk(self, client, db):
+ df = pd.read_csv("tests/fixtures/data/toluene_SDZ54421_2013_2013_v1-0.dat",
+ delimiter=';', comment='#', header=None, names=["datetime", "value", "flags"], parse_dates=["datetime"])
+ df['datetime'] = df['datetime'].dt.tz_localize('UTC')
+ df['version'] = '000001.000000.00000000000000'
+ df['timeseries_id'] = 2
+ df_json=df.to_json(orient='records', date_format='iso')
+ response = client.post("/data/timeseries/bulk/", data=df_json)
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': {'message': 'Data successfully inserted.'}}
+ assert response.json() == expected_resp
"""def test_insert_missing_id(self, client, db):
@@ -656,6 +682,14 @@ class TestApps:
assert response.json() == expected_response
+ def test_get_no_data_with_staging(self, client, db):
+ response = client.get("/data/timeseries_with_staging/id/5")
+ expected_status_code = 404
+ assert response.status_code == expected_status_code
+ expected_response = {'detail': 'Data not found.'}
+ assert response.json() == expected_response
+
+
def test_create_data_record(self, client, db):
response = client.post("/data/timeseries/record/?series_id=2&datetime=2021-08-23%2015:00:00&value=67.3&flag=OK&version=000001.000001.00000000000000")
expected_status_code = 200
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index 6dba596..143d335 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -1133,6 +1133,39 @@ class TestApps:
assert response_json['changelog'][0]['type_of_change'] == 'single value correction in metadata'
+ def test_patch_timeseries_annotations(self, client, db):
+ response = client.patch("/timeseries/id/1?description=changed annotations",
+ json={"timeseries":
+ {"annotations": [{"kind": "User",
+ "text": "some foo",
+ "date_added": "2021-07-27 00:00",
+ "approved": True,
+ "contributor_id":1}]
+ }
+ }
+ )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': { 'message': 'timeseries patched.',
+ 'timeseries_id': 1 }
+ }
+ response_json = response.json()
+ assert response_json == expected_resp
+ response = client.get(f"/timeseries/id/{response_json['detail']['timeseries_id']}")
+ response_json = response.json()
+ # just check special changes
+ response_annotations = [{"kind": "user comment",
+ "text": "some foo",
+ "date_added": "2021-07-27 00:00",
+ "approved": True,
+ "contributor_id":1}
+ ]
+ assert response_json['changelog'][0]['old_value'] == "{'annotations': []}"
+ assert response_json['changelog'][0]['new_value'] == "{'annotations': [{'kind': 'User', 'text': 'some foo', 'date_added': '2021-07-27 00:00', 'approved': True, 'contributor_id': 1}]}"
+ assert response_json['changelog'][0]['author_id'] == 1
+ assert response_json['changelog'][0]['type_of_change'] == 'single value correction in metadata'
+
+
"""def test_get_timeseries_changelog(self, client, db):
response = client.get("/timeseries_changelog/{id}")
expected_status_code = 200
diff --git a/toardb/data/data.py b/toardb/data/data.py
index 9477d49..6f5969f 100644
--- a/toardb/data/data.py
+++ b/toardb/data/data.py
@@ -43,7 +43,7 @@ def get_data(timeseries_id: int, request: Request, db: Session = Depends(get_db)
#get the next available version for one timeseries
@router.get('/data/timeseries/next_version/{timeseries_id}')
-def get_data(timeseries_id: int, request: Request, db: Session = Depends(get_db)):
+def get_version(timeseries_id: int, request: Request, db: Session = Depends(get_db)):
version = crud.get_next_version(db, timeseries_id=timeseries_id, path_params=request.path_params, query_params=request.query_params)
return version
@@ -68,7 +68,7 @@ def get_data_with_staging(timeseries_id: int, flags: str = None, format: str = '
#get map data (for a special variable and timestamp)
@router.get('/data/map/')
-def get_map_data_(variable_id: int = 5, daterange: str = '2023-02-22 12:00,2023-02-22 12:00', db: Session = Depends(get_db)):
+def get_map_data(variable_id: int = 5, daterange: str = '2023-02-22 12:00,2023-02-22 12:00', db: Session = Depends(get_db)):
db_data = crud.get_map_data(db, variable_id=variable_id, daterange=daterange)
if db_data is None:
raise HTTPException(status_code=404, detail="Data not found.")
diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py
index e700ef3..e8cda0b 100644
--- a/toardb/timeseries/crud.py
+++ b/toardb/timeseries/crud.py
@@ -622,6 +622,16 @@ def get_timeseries_role_by_id(db: Session, role_id):
return db.query(models.TimeseriesRole).filter(models.TimeseriesRole.id == role_id).first()
+# is this internal, or should this also go to public REST api?
+def get_timeseries_annotations(db: Session, timeseries_id: int):
+ return db.execute(select([timeseries_timeseries_annotations_table]).where(timeseries_timeseries_annotations_table.c.timeseries_id == timeseries_id))
+
+
+# is this internal, or should this also go to public REST api?
+def get_timeseries_annotation_by_id(db: Session, annotation_id):
+ return db.query(models.TimeseriesAnnotation).filter(models.TimeseriesAnnotation.id == annotation_id).first()
+
+
def create_timeseries(db: Session, timeseries: TimeseriesCreate, author_id: int):
timeseries_dict = timeseries.dict()
# no timeseries can be created, if station_id or variable_id are not found in the database
@@ -886,9 +896,13 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
old_value['contributor_id'] = str(old_role.contact_id)
old_annotations.append(old_value)
old_values['annotations'] = old_annotations
+ new_annotations = []
for a in annotations_data:
- db_annotation = models.StationmetaAnnotation(**a)
+ db_annotation = models.TimeseriesAnnotation(**a)
# check whether annotation is already present in database
+ if not no_log:
+ new_annotations.append(a)
+ db_annotation.kind = get_value_from_str(toardb.toardb.AK_vocabulary,db_annotation.kind)
db_object = get_unique_timeseries_annotation(db, db_annotation.text, db_annotation.contributor_id)
if db_object:
annotation_id = db_object.id
@@ -899,6 +913,8 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
annotation_id = db_annotation.id
db.execute(insert(timeseries_timeseries_annotations_table).values(timeseries_id=timeseries_id, annotation_id=annotation_id))
db.commit()
+ if not no_log:
+ new_values['annotations'] = new_annotations
# add patch to changelog table
if not no_log:
if new_values:
diff --git a/toardb/timeseries/schemas.py b/toardb/timeseries/schemas.py
index 4efb122..a0c8793 100644
--- a/toardb/timeseries/schemas.py
+++ b/toardb/timeseries/schemas.py
@@ -205,17 +205,39 @@ class TimeseriesRoleFields(TimeseriesRoleCreate):
# ======== TimeseriesAnnotation =========
class TimeseriesAnnotationBase(BaseModel):
- id: int = Field(None, description="for internal use only")
- kind: int = Field(..., description="kind of annotation (see controlled vocabulary: Kind Of Annotation)")
+ id: int = None
+ kind: str = Field(..., description="kind of annotation (see controlled vocabulary: Kind Of Annotation)")
text: str = Field(..., description="text of annotation")
date_added: dt.datetime = Field(..., description="timestamp when annotation was added")
approved: bool = Field(..., description="Flag indicating whether the annotation of a time-series has been verified")
contributor_id: int = Field(..., description="ID of contributor who added the annotation")
- timeseries_id: int = Field(..., description="internal timeseries_id to which this annotation belongs")
+
+ @validator('kind')
+ def check_kind(cls, v):
+ return tuple(filter(lambda x: x.value == int(v), toardb.toardb.AK_vocabulary))[0].display_str
+
+
+class TimeseriesAnnotationPatch(BaseModel):
+ kind: int = None
+ text: str = None
+ date_added: dt.datetime = None
+ approved: bool = None
+ contributor_id: int = None
+
+ @validator('kind')
+ def check_kind(cls, v):
+ return tuple(filter(lambda x: x.value == int(v), toardb.toardb.AK_vocabulary))[0].display_str
class TimeseriesAnnotationCreate(TimeseriesAnnotationBase):
- pass
+
+ @validator('kind')
+ def check_kind(cls, v):
+ if tuple(filter(lambda x: x.string == v, toardb.toardb.AK_vocabulary)):
+ return v
+ else:
+ raise ValueError(f"kind of annotation code not known: {v}")
+
class TimeseriesAnnotation(TimeseriesAnnotationBase):
@@ -224,6 +246,7 @@ class TimeseriesAnnotation(TimeseriesAnnotationBase):
class Config:
orm_mode = True
+
# ======== TimeseriesProgramme =========
class TimeseriesProgrammeBase(BaseModel):
@@ -338,9 +361,10 @@ class TimeseriesPatch(TimeseriesCoreCreate):
data_end_date: dt.datetime = None
sampling_height: float = None
# roles: List[TimeseriesRole] = None
+# annotations: List[TimeseriesAnnotationPatch] = None
# just to get things working
roles: list = None
-# annotations: List[TimeseriesAnnotation] = None
+ annotations: list = None
# variable: Variable = None
# station: StationmetaCoreBase = None
# programme: TimeseriesProgramme = None
--
GitLab
From 7c157b854c2ede8b36ff7ce2e644bb73cdfdc014 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Thu, 5 Dec 2024 13:26:57 +0000
Subject: [PATCH 15/36] #168: fields argument is available for endpoint
data/timeseries
---
tests/test_data.py | 291 ++++++++++++++++++++++++++++++++++++++++-
toardb/data/crud.py | 18 ++-
toardb/data/schemas.py | 5 +
toardb/utils/utils.py | 4 +-
4 files changed, 304 insertions(+), 14 deletions(-)
diff --git a/tests/test_data.py b/tests/test_data.py
index 565f073..33dc800 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -33,7 +33,6 @@ from toardb.utils.database import get_db
from datetime import datetime
from unittest.mock import patch
-
# only datetime.now needs to be overridden because otherwise daterange-arguments would be provided as MagicMock-objects!
class FixedDatetime(datetime):
@classmethod
@@ -262,10 +261,7 @@ class TestApps:
# the data/map-endpoint is a special need of the analysis service
def test_get_map_data(self, client, db):
- fixed_time = datetime(2023, 7, 28, 12, 0, 0)
- with patch('toardb.timeseries.crud.dt.datetime') as mock_datetime:
- mock_datetime.now.return_value = fixed_time
- response = client.get("/data/map/?variable_id=7&daterange=2012-12-16T21:00,2012-12-17T06:00")
+ response = client.get("/data/map/?variable_id=7&daterange=2012-12-16T21:00,2012-12-17T06:00")
expected_status_code = 200
assert response.status_code == expected_status_code
expected_resp = [{'timeseries_id': 1, 'value': 21.581},
@@ -281,6 +277,291 @@ class TestApps:
assert response.json() == expected_resp
+ def test_get_data_with_fields(self, client, db):
+ fixed_time = datetime(2023, 7, 28, 12, 0, 0)
+ with patch('toardb.timeseries.crud.dt.datetime') as mock_datetime:
+ mock_datetime.now.return_value = fixed_time
+ response = client.get("/data/timeseries/1?fields=datetime,value")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'metadata': {'id': 1,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata': {},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 2,
+ 'codes': ['SDZ54421'],
+ 'name': 'Shangdianzi',
+ 'coordinates': {'lat': 40.65,
+ 'lng': 117.106,
+ 'alt': 293.9},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China',
+ 'state': 'Beijing Shi',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {'dummy_info': 'Here is some '
+ 'more information about the station'},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': None,
+ 'changelog': []},
+ 'variable': {'name': 'toluene',
+ 'longname': 'toluene',
+ 'displayname': 'Toluene',
+ 'cf_standardname': 'mole_fraction_of_toluene_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'C7H8',
+ 'id': 7},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 4,
+ 'person': None,
+ 'organisation': {'id': 1,
+ 'name': 'UBA',
+ 'longname': 'Umweltbundesamt',
+ 'kind': 'government',
+ 'city': 'Dessau-Roßlau',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'contact_url': 'mailto:immission@uba.de'}}}],
+ 'changelog': None,
+ 'citation': 'Umweltbundesamt: time series of toluene at '
+ 'Shangdianzi, accessed from the TOAR database on '
+ '2023-07-28 12:00:00',
+ 'license': 'This data is published under a Creative Commons '
+ 'Attribution 4.0 International (CC BY 4.0). '
+ 'https://creativecommons.org/licenses/by/4.0/'},
+ 'data': [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581},
+ {'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734},
+ {'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734},
+ {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848},
+ {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696},
+ {'datetime': '2012-12-17T02:00:00+00:00', 'value': 11.772},
+ {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734},
+ {'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62},
+ {'datetime': '2012-12-17T05:00:00+00:00', 'value': 15.696},
+ {'datetime': '2012-12-17T06:00:00+00:00', 'value': 5.886}]
+ }
+ assert response.json() == expected_resp
+
+
+ def test_get_data_with_fields_limited(self, client, db):
+ fixed_time = datetime(2023, 7, 28, 12, 0, 0)
+ with patch('toardb.timeseries.crud.dt.datetime') as mock_datetime:
+ mock_datetime.now.return_value = fixed_time
+ response = client.get("/data/timeseries/1?fields=datetime,value&limit=4")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'metadata': {'id': 1,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata': {},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 2,
+ 'codes': ['SDZ54421'],
+ 'name': 'Shangdianzi',
+ 'coordinates': {'lat': 40.65,
+ 'lng': 117.106,
+ 'alt': 293.9},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China',
+ 'state': 'Beijing Shi',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {'dummy_info': 'Here is some '
+ 'more information about the station'},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': None,
+ 'changelog': []},
+ 'variable': {'name': 'toluene',
+ 'longname': 'toluene',
+ 'displayname': 'Toluene',
+ 'cf_standardname': 'mole_fraction_of_toluene_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'C7H8',
+ 'id': 7},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 4,
+ 'person': None,
+ 'organisation': {'id': 1,
+ 'name': 'UBA',
+ 'longname': 'Umweltbundesamt',
+ 'kind': 'government',
+ 'city': 'Dessau-Roßlau',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'contact_url': 'mailto:immission@uba.de'}}}],
+ 'changelog': None,
+ 'citation': 'Umweltbundesamt: time series of toluene at '
+ 'Shangdianzi, accessed from the TOAR database on '
+ '2023-07-28 12:00:00',
+ 'license': 'This data is published under a Creative Commons '
+ 'Attribution 4.0 International (CC BY 4.0). '
+ 'https://creativecommons.org/licenses/by/4.0/'},
+ 'data': [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581},
+ {'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734},
+ {'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734},
+ {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848}]
+ }
+ assert response.json() == expected_resp
+
+
+ def test_get_data_as_csv_with_fields(self, client, db):
+ fixed_time = datetime(2023, 7, 28, 12, 0, 0)
+ with patch('toardb.timeseries.crud.dt.datetime') as mock_datetime:
+ mock_datetime.now.return_value = fixed_time
+ response = client.get("/data/timeseries/1?format=csv&fields=datetime,value")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = ''.join(['#{\n',
+ '# "id": 1,\n',
+ '# "label": "CMA",\n',
+ '# "order": 1,\n',
+ '# "sampling_frequency": "hourly",\n',
+ '# "aggregation": "mean",\n',
+ '# "data_start_date": "2003-09-07T15:30:00+00:00",\n',
+ '# "data_end_date": "2016-12-31T14:30:00+00:00",\n',
+ '# "data_origin": "instrument",\n',
+ '# "data_origin_type": "measurement",\n',
+ '# "provider_version": "N/A",\n',
+ '# "sampling_height": 7.0,\n',
+ '# "additional_metadata": {},\n',
+ '# "data_license_accepted": null,\n',
+ '# "dataset_approved_by_provider": null,\n',
+ '# "doi": "",\n',
+ '# "coverage": -1.0,\n',
+ '# "station": {\n',
+ '# "id": 2,\n',
+ '# "codes": [\n',
+ '# "SDZ54421"\n',
+ '# ],\n',
+ '# "name": "Shangdianzi",\n',
+ '# "coordinates": {\n',
+ '# "lat": 40.65,\n',
+ '# "lng": 117.106,\n',
+ '# "alt": 293.9\n',
+ '# },\n',
+ '# "coordinate_validation_status": "not checked",\n',
+ '# "country": "China",\n',
+ '# "state": "Beijing Shi",\n',
+ '# "type": "unknown",\n',
+ '# "type_of_area": "unknown",\n',
+ '# "timezone": "Asia/Shanghai",\n',
+ '# "additional_metadata": {\n',
+ '# "dummy_info": "Here is some more information about the station"\n',
+ '# },\n',
+ '# "roles": [],\n',
+ '# "annotations": [],\n',
+ '# "aux_images": [],\n',
+ '# "aux_docs": [],\n',
+ '# "aux_urls": [],\n',
+ '# "globalmeta": null,\n',
+ '# "changelog": []\n',
+ '# },\n',
+ '# "variable": {\n',
+ '# "name": "toluene",\n',
+ '# "longname": "toluene",\n',
+ '# "displayname": "Toluene",\n',
+ '# "cf_standardname": "mole_fraction_of_toluene_in_air",\n',
+ '# "units": "nmol mol-1",\n',
+ '# "chemical_formula": "C7H8",\n',
+ '# "id": 7\n',
+ '# },\n',
+ '# "programme": {\n',
+ '# "id": 0,\n',
+ '# "name": "",\n',
+ '# "longname": "",\n',
+ '# "homepage": "",\n',
+ '# "description": ""\n',
+ '# },\n',
+ '# "roles": [\n',
+ '# {\n',
+ '# "id": 2,\n',
+ '# "role": "resource provider",\n',
+ '# "status": "active",\n',
+ '# "contact": {\n',
+ '# "id": 4,\n',
+ '# "person": null,\n',
+ '# "organisation": {\n',
+ '# "id": 1,\n',
+ '# "name": "UBA",\n',
+ '# "longname": "Umweltbundesamt",\n',
+ '# "kind": "government",\n',
+ '# "city": "Dessau-Roßlau",\n',
+ '# "postcode": "06844",\n',
+ '# "street_address": "Wörlitzer Platz 1",\n',
+ '# "country": "Germany",\n',
+ '# "homepage": "https://www.umweltbundesamt.de",\n',
+ '# "contact_url": "mailto:immission@uba.de"\n',
+ '# }\n',
+ '# }\n',
+ '# }\n',
+ '# ],\n',
+ '# "annotations": null,\n',
+ '# "changelog": null,\n',
+ '# "citation": "Umweltbundesamt: time series of toluene at Shangdianzi, accessed from the TOAR database on 2023-07-28 12:00:00",\n',
+ '# "attribution": null,\n',
+ '# "license": "This data is published under a Creative Commons Attribution 4.0 International (CC BY 4.0). https://creativecommons.org/licenses/by/4.0/"\n',
+ '#}\n',
+ 'datetime,value\n',
+ '2012-12-16 21:00:00+00:00,21.581\n',
+ '2012-12-16 22:00:00+00:00,13.734\n',
+ '2012-12-16 23:00:00+00:00,13.734\n',
+ '2012-12-17 00:00:00+00:00,7.848\n',
+ '2012-12-17 01:00:00+00:00,15.696\n',
+ '2012-12-17 02:00:00+00:00,11.772\n',
+ '2012-12-17 03:00:00+00:00,13.734\n',
+ '2012-12-17 04:00:00+00:00,19.62\n',
+ '2012-12-17 05:00:00+00:00,15.696\n',
+ '2012-12-17 06:00:00+00:00,5.886'])
+ assert response.text == expected_resp
+
+
def test_get_no_data_with_variable_and_timerage(self, client, db):
# see: https://gitlab.jsc.fz-juelich.de/esde/toar-data/toardb_fastapi/-/issues/171
response = client.get("/data/map/?variable_id=25&daterange=2012-12-16T21:00,2012-12-17T06:00")
diff --git a/toardb/data/crud.py b/toardb/data/crud.py
index d3d3aab..1d36bb5 100644
--- a/toardb/data/crud.py
+++ b/toardb/data/crud.py
@@ -127,19 +127,25 @@ def get_data(db: Session, timeseries_id: int, path_params, query_params):
flags = None
limit, offset, fields, format, filters = create_filter(query_params, "data")
d_filter = filters["d_filter"]
+ fields_list = []
+ if fields:
+ fields_list = fields.split(',')
+ columns = ( [getattr(models.Data, field) for field in fields_list]
+ if fields_list
+ else list(models.Data.__table__.columns) )
except KeyError as e:
status_code=400
return JSONResponse(status_code=status_code, content=str(e))
if flags:
filter_string = create_filter_from_flags(flags)
- data = db.query(models.Data).filter(models.Data.timeseries_id == timeseries_id). \
+ data = db.query(*columns).filter(models.Data.timeseries_id == timeseries_id). \
filter(text(filter_string)). \
filter(text(d_filter)). \
- order_by(models.Data.datetime).all()
+ order_by(models.Data.datetime).limit(limit).all()
else:
- data = db.query(models.Data).filter(models.Data.timeseries_id == timeseries_id). \
+ data = db.query(*columns).filter(models.Data.timeseries_id == timeseries_id). \
filter(text(d_filter)). \
- order_by(models.Data.datetime).all()
+ order_by(models.Data.datetime).limit(limit).all()
# get advantages from pydantic, but without having another call of the REST API
# (especially needed for testing with pytest!)
metadata = get_timeseries_meta(timeseries_id)
@@ -154,9 +160,9 @@ def get_data(db: Session, timeseries_id: int, path_params, query_params):
# start with metadata
content = '#' + metadata.json(indent=4, ensure_ascii=False).replace('\n', '\n#') + '\n'
# add header
- content += ','.join(column.name for column in models.Data.__mapper__.columns) + '\n'
+ content += ','.join(column.name for column in columns) + '\n'
# now the data
- content += '\n'.join(','.join(f"{getattr(curr, column.name)}" for column in models.Data.__mapper__.columns) for curr in data)
+ content += '\n'.join(','.join(f"{getattr(curr, column.name)}" for column in columns) for curr in data)
return Response(content=content, media_type="text/csv")
else:
status_code=400
diff --git a/toardb/data/schemas.py b/toardb/data/schemas.py
index d4b410e..7ee0008 100644
--- a/toardb/data/schemas.py
+++ b/toardb/data/schemas.py
@@ -66,6 +66,11 @@ class DataCreate(DataBase):
class Data(DataBase):
+ datetime: dt.datetime = None
+ value: float = None
+ flags: str = None
+ timeseries_id: int = None
+ version: str = None
class Config:
orm_mode = True
diff --git a/toardb/utils/utils.py b/toardb/utils/utils.py
index 71316d8..5bdc457 100644
--- a/toardb/utils/utils.py
+++ b/toardb/utils/utils.py
@@ -165,8 +165,6 @@ def create_filter(query_params, endpoint):
# fields and format are no filter options
fields = query_params.get("fields", None)
- if fields and endpoint == "data":
- raise KeyError(f"An unknown argument was received: fields.")
format = query_params.get("format", 'json')
allowed_params = allrel_params.copy()
@@ -178,7 +176,7 @@ def create_filter(query_params, endpoint):
elif endpoint in {'search'}:
allowed_params |= gis_params | core_params | global_params | timeseries_params | roles_params | ambig_params
elif endpoint in {'data'}:
- allowed_params = {"limit", "offset" } | data_params | profiling_params
+ allowed_params |= data_params | profiling_params
elif endpoint in {'variables'}:
allowed_params |= variable_params
elif endpoint in {'persons'}:
--
GitLab
From 6c9d480acade486f1994e7d658e7dc1674508f89 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 6 Dec 2024 15:10:02 +0000
Subject: [PATCH 16/36] preparation for testing the (re-)integrated AAI;
started strictly separating operational and testing database settings;
corrected outdated DataFrame access
---
tests/test_data.py | 79 ++++++++++++-----------------
tests/test_stationmeta.py | 103 +++++++++++++++++++++-----------------
tests/test_timeseries.py | 100 +++++++++++++++++++++---------------
toardb/data/crud.py | 6 +--
toardb/test_base.py | 38 ++++++++++++++
toardb/utils/database.py | 4 +-
6 files changed, 191 insertions(+), 139 deletions(-)
diff --git a/tests/test_data.py b/tests/test_data.py
index 33dc800..2e83788 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -19,16 +19,12 @@ from toardb.auth_user.models import AuthUser
from toardb.test_base import (
client,
get_test_db,
+ override_dependency,
create_test_database,
url,
get_test_engine,
test_db_session as db,
)
-from toardb.utils.utils import (
- get_admin_access_rights,
- get_data_change_access_rights
-)
-from toardb.utils.database import get_db
# for mocking datetime.now(timezone.utc)
from datetime import datetime
from unittest.mock import patch
@@ -40,24 +36,6 @@ class FixedDatetime(datetime):
return datetime(2023, 7, 28, 12, 0, 0)
-async def override_dependency(request: Request):
- db = next(get_db())
- db_user = db.query(AuthUser).filter(AuthUser.email == "s.schroeder@fz-juelich.de").first()
- if db_user:
- auth_user_id = db_user.id
- else: # pragma: no cover
- # the user needs to be added to the database!
- pass
- access_dict = { "status_code": 200,
- "user_name": "Sabine Schröder",
- "user_email": "s.schroeder@fz-juelich.de",
- "auth_user_id": 1 }
- return access_dict
-
-app.dependency_overrides[get_admin_access_rights] = override_dependency
-app.dependency_overrides[get_data_change_access_rights] = override_dependency
-
-
class TestApps:
def setup(self):
self.application_url = "/data/"
@@ -573,25 +551,29 @@ class TestApps:
assert response.json() == expected_response
-# def test_insert_new_without_credits(self):
-#? response = client.post("/data/timeseries/")
-# expected_status_code=401
-# assert response.status_code == expected_status_code
-#? expected_resp = ...
-# assert response.json() == expected_resp
-
+ def test_insert_new_wrong_credentials(self, client, db):
+ response = client.post("/data/timeseries/?toarqc_config_type=standard",
+ files={"file": open("tests/fixtures/data/toluene_SDZ54421_2013_2013_v1-0.dat", "rb")},
+ headers={"email": "j.doe@fz-juelich.de"})
+ expected_status_code=401
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': 'Unauthorized.'}
+ assert response.json() == expected_resp
-# def test_insert_new_wrong_credits(self):
-#? response = client.post("/data/timeseries/")
-# expected_status_code = 401
-# assert response.status_code == expected_status_code
-#? expected_resp = ...
-# assert response.json() == expected_resp
+ def test_insert_new_without_credentials(self, client, db):
+ response = client.post("/data/timeseries/?toarqc_config_type=standard",
+ files={"file": open("tests/fixtures/data/toluene_SDZ54421_2013_2013_v1-0.dat", "rb")})
+ expected_status_code=401
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': 'Unauthorized.'}
+ assert response.json() == expected_resp
def test_insert_new(self, client, db):
- response = client.post("/data/timeseries/?toarqc_config_type=standard", files={"file": open("tests/fixtures/data/toluene_SDZ54421_2013_2013_v1-0.dat", "rb")})
+ response = client.post("/data/timeseries/?toarqc_config_type=standard",
+ files={"file": open("tests/fixtures/data/toluene_SDZ54421_2013_2013_v1-0.dat", "rb")},
+ headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 200
assert response.status_code == expected_status_code
expected_resp = {'detail': {'message': 'Data successfully inserted.'}}
@@ -599,7 +581,9 @@ class TestApps:
def test_insert_duplicate(self, client, db):
- response = client.post("/data/timeseries/?toarqc_config_type=standard", files={"file": open("tests/fixtures/data/toluene_SDZ54421_2012_2012_v1-0.dat", "rb")})
+ response = client.post("/data/timeseries/?toarqc_config_type=standard",
+ files={"file": open("tests/fixtures/data/toluene_SDZ54421_2012_2012_v1-0.dat", "rb")},
+ headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 400
assert response.status_code == expected_status_code
expected_resp = {'detail': 'Data for timeseries already registered.'}
@@ -613,7 +597,7 @@ class TestApps:
df['version'] = '000001.000000.00000000000000'
df['timeseries_id'] = 2
df_json=df.to_json(orient='records', date_format='iso')
- response = client.post("/data/timeseries/bulk/", data=df_json)
+ response = client.post("/data/timeseries/bulk/", data=df_json, headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 200
assert response.status_code == expected_status_code
expected_resp = {'detail': {'message': 'Data successfully inserted.'}}
@@ -972,7 +956,8 @@ class TestApps:
def test_create_data_record(self, client, db):
- response = client.post("/data/timeseries/record/?series_id=2&datetime=2021-08-23%2015:00:00&value=67.3&flag=OK&version=000001.000001.00000000000000")
+ response = client.post("/data/timeseries/record/?series_id=2&datetime=2021-08-23%2015:00:00&value=67.3&flag=OK&version=000001.000001.00000000000000",
+ headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 200
assert response.status_code == expected_status_code
expected_response = 'Data successfully inserted.'
@@ -988,7 +973,9 @@ class TestApps:
def test_patch_data(self, client, db):
- response = client.patch("/data/timeseries/?description=test patch&version=000002.000000.00000000000000", files={"file": open("tests/fixtures/data/toluene_SDZ54421_2012_2012_v2-0.dat", "rb")})
+ response = client.patch("/data/timeseries/?description=test patch&version=000002.000000.00000000000000",
+ files={"file": open("tests/fixtures/data/toluene_SDZ54421_2012_2012_v2-0.dat", "rb")},
+ headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 200
assert response.status_code == expected_status_code
expected_response = {'detail': {'message': 'Data successfully inserted.'}}
@@ -1020,8 +1007,8 @@ class TestApps:
{"datetime": "2012-12-17T01:00:00+00:00", "value": 15.696, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"},
{"datetime": "2012-12-17T02:00:00+00:00", "value": 11.772, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"},
{"datetime": "2012-12-17T03:00:00+00:00", "value": 13.734, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"},
- {"datetime": "2012-12-17T04:00:00+00:00", "value": 19.62, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"}]'''
- )
+ {"datetime": "2012-12-17T04:00:00+00:00", "value": 19.62, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"}]''',
+ headers={"email": "s.schroeder@fz-juelich.de"} )
expected_status_code = 200
assert response.status_code == expected_status_code
expected_response = {'detail': {'message': 'Data successfully inserted.'}}
@@ -1134,13 +1121,13 @@ class TestApps:
def test_patch_bulk_data2(self, client, db):
- response = client.patch("/data/timeseries/bulk/?pwd=None&toarqc_config_type=realtime&no_archive=True&description=NOLOG&version=000000.000001.20230208144921&force=True",
+ response = client.patch("/data/timeseries/bulk/?toarqc_config_type=realtime&no_archive=True&description=NOLOG&version=000000.000001.20230208144921&force=True",
data='''[{"datetime": "2012-12-17T00:00:00+00:00", "value": 7.848, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"},
{"datetime": "2012-12-17T01:00:00+00:00", "value": 15.696, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"},
{"datetime": "2012-12-17T02:00:00+00:00", "value": 11.772, "flags": 0, "timeseries_id": 1, "version": "000002.000000.00000000000000"},
{"datetime": "2012-12-17T03:00:00+00:00", "value": 13.734, "flags": 0, "timeseries_id": 2, "version": "000002.000000.00000000000000"},
- {"datetime": "2012-12-17T04:00:00+00:00", "value": 19.62, "flags": 0, "timeseries_id": 2, "version": "000002.000000.00000000000000"}]'''
- )
+ {"datetime": "2012-12-17T04:00:00+00:00", "value": 19.62, "flags": 0, "timeseries_id": 2, "version": "000002.000000.00000000000000"}]''',
+ headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 200
assert response.status_code == expected_status_code
expected_response = {'detail': {'message': 'Data successfully inserted.'}}
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index 608ce9a..e98d180 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -25,34 +25,12 @@ from toardb.contacts.models import Person, Organisation, Contact
from toardb.test_base import (
client,
get_test_db,
+ override_dependency,
create_test_database,
url,
get_test_engine,
test_db_session as db,
)
-from toardb.utils.utils import (
- get_admin_access_rights,
- get_station_md_change_access_rights
-)
-
-
-# try "tesing dependencies with overrides"
-# https://fastapi.tiangolo.com/advanced/testing-dependencies/
-
-# now try: "Advanced Dependencies"
-# https://fastapi.tiangolo.com/advanced/advanced-dependencies/
-# --> does not work!
-
-
-async def override_dependency(request: Request):
- access_dict = { "status_code": 200,
- "user_name": "Sabine Schröder",
- "user_email": "s.schroeder@fz-juelich.de",
- "auth_user_id": 1 }
- return access_dict
-
-app.dependency_overrides[get_admin_access_rights] = override_dependency
-app.dependency_overrides[get_station_md_change_access_rights] = override_dependency
class TestApps:
@@ -701,20 +679,39 @@ class TestApps:
# 2. tests creating station metadata
-# def test_insert_new_without_credits(self):
-#? response = client.post("/stationmeta/")
-# expected_status_code=401
-# assert response.status_code == expected_status_code
-#? expected_resp = ...
-# assert response.json() == expected_resp
+
+ def test_insert_new_wrong_credentials(self, client, db):
+ response = client.post("/stationmeta/",
+ json={"stationmeta":
+ {"codes":["ttt3","ttt4"],
+ "name":"Test_China","coordinates":{"lat":37.256,"lng":117.106,"alt":1534.0},
+ "coordinate_validation_status": "NotChecked",
+ "country":"CN","state":"Shandong Sheng",
+ "type":"Unknown","type_of_area":"Unknown","timezone":"Asia/Shanghai",
+ "additional_metadata": "{}" }
+ },
+ headers={"email": "j.doe@fz-juelich.de"}
+ )
+ expected_status_code=401
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': 'Unauthorized.'}
+ assert response.json() == expected_resp
-# def test_insert_new_wrong_credits(self):
-#? response = client.post("/stationmeta/")
-# expected_status_code = 401
-# assert response.status_code == expected_status_code
-#? expected_resp = ...
-# assert response.json() == expected_resp
+ def test_insert_new_without_credentials(self, client, db):
+ response = client.post("/stationmeta/",
+ json={"stationmeta":
+ {"codes":["ttt3","ttt4"],
+ "name":"Test_China","coordinates":{"lat":37.256,"lng":117.106,"alt":1534.0},
+ "coordinate_validation_status": "NotChecked",
+ "country":"CN","state":"Shandong Sheng",
+ "type":"Unknown","type_of_area":"Unknown","timezone":"Asia/Shanghai",
+ "additional_metadata": "{}" }
+ })
+ expected_status_code=401
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': 'Unauthorized.'}
+ assert response.json() == expected_resp
def test_insert_new(self, client, db):
@@ -726,7 +723,8 @@ class TestApps:
"country":"CN","state":"Shandong Sheng",
"type":"Unknown","type_of_area":"Unknown","timezone":"Asia/Shanghai",
"additional_metadata": "{}" }
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -746,7 +744,8 @@ class TestApps:
"roles": [{"role": "PointOfContact", "contact_id": 3, "status": "Active"},
{"role": "Originator", "contact_id": 1, "status": "Active"}],
"additional_metadata": "{}" }
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -769,7 +768,8 @@ class TestApps:
"approved": True,
"contributor_id":1}],
"additional_metadata": "{}" }
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -787,7 +787,8 @@ class TestApps:
"country":"CN","state":"Shandong Sheng",
"type":"Unknown","type_of_area":"Unknown","timezone":"Asia/Shanghai",
"additional_metadata":"{}"},
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 444
assert response.status_code == expected_status_code
@@ -807,7 +808,8 @@ class TestApps:
"country":"CN","state":"Shandong Sheng",
"type":"Unknown","type_of_area":"Unknown","timezone":"Asia/Shanghai",
"additional_metadata":"{}"}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 443
assert response.status_code == expected_status_code
@@ -822,7 +824,8 @@ class TestApps:
response = client.patch("/stationmeta/-1",
json={"stationmeta":
{"name":"TTTT95TTTT"}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 404
assert response.status_code == expected_status_code
@@ -834,7 +837,8 @@ class TestApps:
response = client.patch("/stationmeta/-1?description=changing station name",
json={"stationmeta":
{"name":"TTTT95TTTT"}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 404
assert response.status_code == expected_status_code
@@ -846,7 +850,8 @@ class TestApps:
response = client.patch("/stationmeta/SDZ54421?description=changing station name",
json={"stationmeta":
{"name":"TTTT95TTTT"}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -867,7 +872,8 @@ class TestApps:
response = client.patch("/stationmeta/SDZ54421?description=changing global metadata",
json={"stationmeta":
{"globalmeta": {"climatic_zone_year2016": "WarmTemperateMoist"}}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -894,7 +900,8 @@ class TestApps:
"landcover_description_25km_year2012": "TreeNeedleleavedEvergreenClosedToOpen: 100 %",
"dominant_ecoregion_year2017": "Guianansavanna",
"ecoregion_description_25km_year2017": "Guianansavanna: 90 %, Miskitopineforests: 10 %"}}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -926,7 +933,8 @@ class TestApps:
def test_delete_roles_from_stationmeta(self, client, db):
- response = client.patch("/stationmeta/delete_field/China11?field=roles")
+ response = client.patch("/stationmeta/delete_field/China11?field=roles",
+ headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 200
assert response.status_code == expected_status_code
expected_resp = {'codes': ['China11'],
@@ -949,7 +957,8 @@ class TestApps:
def test_delete_field_station_not_found(self, client, db):
- response = client.patch("/stationmeta/delete_field/China22?field=timezone")
+ response = client.patch("/stationmeta/delete_field/China22?field=timezone",
+ headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 404
assert response.status_code == expected_status_code
expected_resp = {'detail': 'Station for deleting field not found.'}
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index 143d335..7270131 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -4,8 +4,6 @@
import pytest
import json
from sqlalchemy import insert
-from fastapi import Request
-from toardb.toardb import app
from toardb.timeseries.models import Timeseries, timeseries_timeseries_roles_table
from toardb.timeseries.models_programme import TimeseriesProgramme
from toardb.timeseries.models_role import TimeseriesRole
@@ -18,31 +16,17 @@ from toardb.auth_user.models import AuthUser
from toardb.test_base import (
client,
get_test_db,
+ override_dependency,
create_test_database,
url,
get_test_engine,
test_db_session as db,
)
-from toardb.utils.utils import (
- get_admin_access_rights,
- get_timeseries_md_change_access_rights
-)
# for mocking datetime.now(timezone.utc)
from datetime import datetime
from unittest.mock import patch
-async def override_dependency(request: Request):
- access_dict = { "status_code": 200,
- "user_name": "Sabine Schröder",
- "user_email": "s.schroeder@fz-juelich.de",
- "auth_user_id": 1 }
- return access_dict
-
-app.dependency_overrides[get_admin_access_rights] = override_dependency
-app.dependency_overrides[get_timeseries_md_change_access_rights] = override_dependency
-
-
class TestApps:
def setup(self):
self.application_url = "/timeseries/"
@@ -442,21 +426,45 @@ class TestApps:
assert response.json() == expected_resp
-# def test_insert_new_without_credits(self):
-#? response = client.post("/timeseries/")
-# expected_status_code=401
-# assert response.status_code == expected_status_code
-#? expected_resp = ...
-# assert response.json() == expected_resp
+ def test_insert_new_wrong_credentials(self, client, db):
+ response = client.post("/timeseries/",
+ json={"timeseries":
+ {"label": "CMA2", "order": 1,
+ "sampling_frequency": "Hourly", "aggregation": "Mean", "data_origin_type": "Measurement",
+ "data_start_date": "2003-09-07T15:30:00+02:00",
+ "data_end_date": "2016-12-31T14:30:00+01:00",
+ 'coverage': -1.0,
+ "data_origin": "Instrument", "sampling_height": 7.0,
+ "station_id": 2, "variable_id": 7,
+ "additional_metadata":"{}"
+ }
+ },
+ headers={"email": "j.doe@fz-juelich.de"}
+ )
+ expected_status_code=401
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': 'Unauthorized.'}
+ assert response.json() == expected_resp
+ def test_insert_new_without_credentials(self, client, db):
+ response = client.post("/timeseries/",
+ json={"timeseries":
+ {"label": "CMA2", "order": 1,
+ "sampling_frequency": "Hourly", "aggregation": "Mean", "data_origin_type": "Measurement",
+ "data_start_date": "2003-09-07T15:30:00+02:00",
+ "data_end_date": "2016-12-31T14:30:00+01:00",
+ 'coverage': -1.0,
+ "data_origin": "Instrument", "sampling_height": 7.0,
+ "station_id": 2, "variable_id": 7,
+ "additional_metadata":"{}"
+ }
+ })
+ expected_status_code=401
+ assert response.status_code == expected_status_code
+ expected_resp = {'detail': 'Unauthorized.'}
+ assert response.json() == expected_resp
-# def test_insert_new_wrong_credits(self):
-#? response = client.post("/timeseries/")
-# expected_status_code = 401
-# assert response.status_code == expected_status_code
-#? expected_resp = ...
-# assert response.json() == expected_resp
def test_insert_new_with_roles(self, client, db):
add_meta_dict = {"absorption_cross_section": "Hearn1961", "measurement_method": "uv_abs"}
@@ -470,11 +478,11 @@ class TestApps:
"data_origin": "Instrument", "sampling_height": 7.0,
"station_id": 2, "variable_id": 5,
"additional_metadata": json.dumps(add_meta_dict),
-# "additional_metadata": {"absorption_cross_section": 0, "measurement_method": "uv_abs"},
"roles": [{"role": "PointOfContact", "contact_id": 3, "status": "Active"},
{"role": "Originator", "contact_id": 1, "status": "Active"}]
}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -505,7 +513,8 @@ class TestApps:
"roles": [{"role": "PointOfContact", "contact_id": 3, "status": "Active"},
{"role": "Originator", "contact_id": 1, "status": "Active"}]
}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 440
assert response.status_code == expected_status_code
@@ -528,7 +537,8 @@ class TestApps:
"roles": [{"role": "PointOfContact", "contact_id": 3, "status": "Active"},
{"role": "Originator", "contact_id": 1, "status": "Active"}]
}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -552,7 +562,8 @@ class TestApps:
{"role": "Originator", "contact_id": 1, "status": "Active"},
{"role": "ResourceProvider", "contact_id": 1, "status": "Active"}]
}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 442
assert response.status_code == expected_status_code
@@ -576,7 +587,8 @@ class TestApps:
{"role": "Originator", "contact_id": 1, "status": "Active"},
{"role": "ResourceProvider", "contact_id": 4, "status": "Active"}]
}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 443
assert response.status_code == expected_status_code
@@ -1007,7 +1019,8 @@ class TestApps:
response = client.patch("/timeseries/id/1",
json={"timeseries":
{"sampling_frequency":"Daily"}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 404
assert response.status_code == expected_status_code
@@ -1019,7 +1032,8 @@ class TestApps:
response = client.patch("/timeseries/id/-1?description=changed sampling_frequency",
json={"timeseries":
{"sampling_frequency":"Daily"}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 404
assert response.status_code == expected_status_code
@@ -1031,7 +1045,8 @@ class TestApps:
response = client.patch("/timeseries/id/1?description=changed sampling_frequency",
json={"timeseries":
{"sampling_frequency": "Daily"}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -1057,7 +1072,8 @@ class TestApps:
"aggregation": "MeanOf4Samples",
"data_origin": "COSMOREA6",
"data_origin_type": "Model"}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -1081,7 +1097,8 @@ class TestApps:
json={"timeseries":
{"roles": [{"role": "ResourceProvider", "contact_id": 5, "status": "Active"}]
}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
@@ -1142,7 +1159,8 @@ class TestApps:
"approved": True,
"contributor_id":1}]
}
- }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
)
expected_status_code = 200
assert response.status_code == expected_status_code
diff --git a/toardb/data/crud.py b/toardb/data/crud.py
index 1d36bb5..14ecda9 100644
--- a/toardb/data/crud.py
+++ b/toardb/data/crud.py
@@ -288,7 +288,7 @@ def create_data_record(db: Session, engine: Engine,
( 'OK', 'Erroneous') : 'ErroneousPreliminaryFlagged1',
( 'Questionable', 'Erroneous') : 'ErroneousPreliminaryFlagged2',
( 'Erroneous', 'Erroneous') : 'Erroneous_Preliminary_Confirmed'}
- combined_flag = combined_flag_matrix[(flag,result['flags'][0])]
+ combined_flag = combined_flag_matrix[(flag,result['flags'].iloc[0])]
data_dict["flags"] = get_value_from_str(toardb.toardb.DF_vocabulary,combined_flag.strip())
data = models.Data(**data_dict)
db.rollback()
@@ -529,7 +529,7 @@ def create_bulk_data(db: Session, engine: Engine, bulk: List[schemas.DataCreate]
df = pd.DataFrame([x.dict() for x in bulk]).set_index("datetime")
# bulk data: to be able to do at least a range test, all data needs to be from the same parameter
# variable_name is therefore determined for the first entry of the dataframe
- timeseries = get_timeseries(db=db,timeseries_id=int(df['timeseries_id'][0]))
+ timeseries = get_timeseries(db=db,timeseries_id=int(df['timeseries_id'].iloc[0]))
# again problems with converted coordinates!
db.rollback()
variable = get_variable(db=db, variable_id=timeseries.variable_id)
@@ -657,7 +657,7 @@ def patch_bulk_data(db: Session, engine: Engine, description: str, version: str,
df = pd.DataFrame([x.dict() for x in bulk]).set_index("datetime")
# bulk data: to be able to do at least a range test, all data needs to be from the same parameter
# variable_name is therefore determined for the first entry of the dataframe
- timeseries_id = int(df['timeseries_id'][0])
+ timeseries_id = int(df['timeseries_id'].iloc[0])
timeseries = get_timeseries(db=db,timeseries_id=timeseries_id)
variable = get_variable(db=db, variable_id=timeseries.variable_id)
variable_name = variable.name
diff --git a/toardb/test_base.py b/toardb/test_base.py
index ad331ce..e30e568 100644
--- a/toardb/test_base.py
+++ b/toardb/test_base.py
@@ -8,10 +8,18 @@ from sqlalchemy import create_engine
from sqlalchemy.engine import Engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy_utils import database_exists, create_database, drop_database
+from fastapi import Request
from toardb.base import Base
from toardb.toardb import app
+from toardb.auth_user.models import AuthUser
from toardb.utils.database import DATABASE_URL, get_db, get_engine
+from toardb.utils.utils import (
+ get_admin_access_rights,
+ get_station_md_change_access_rights,
+ get_timeseries_md_change_access_rights,
+ get_data_change_access_rights
+)
url = "postgresql://postgres:postgres@postgres:5432/postgres"
_db_conn = create_engine(url,pool_pre_ping=True, pool_size=32, max_overflow=128)
@@ -30,6 +38,36 @@ def get_test_db():
test_db.close()
+async def override_dependency(request: Request):
+ db = next(get_test_db())
+ email = request.headers.get('email')
+ db_user = db.query(AuthUser).filter(AuthUser.email == email).first()
+ # status_code will be taken from the AAI (here: faked)
+ status_code = 401
+ if db_user:
+ # status_code will be taken from the AAI (here: faked)
+ status_code = 200
+ access_dict = { "status_code": status_code,
+ "user_name": "Sabine Schröder",
+ "user_email": email,
+ "auth_user_id": db_user.id }
+ else: # pragma: no cover
+ # the user needs to be added to the database!
+ # (maybe users already have the credentials (in the AAI),
+ # but they also need a permanent auth_user_id related to the TOAR database)
+ access_dict = { "status_code": status_code,
+ "user_name": "Something from AAI",
+ "user_email": email,
+ "auth_user_id": -1 }
+ return access_dict
+
+
+app.dependency_overrides[get_admin_access_rights] = override_dependency
+app.dependency_overrides[get_station_md_change_access_rights] = override_dependency
+app.dependency_overrides[get_timeseries_md_change_access_rights] = override_dependency
+app.dependency_overrides[get_data_change_access_rights] = override_dependency
+
+
@pytest.fixture(scope="session", autouse=True)
def create_test_database():
"""
diff --git a/toardb/utils/database.py b/toardb/utils/database.py
index 8c9e9f8..043780b 100644
--- a/toardb/utils/database.py
+++ b/toardb/utils/database.py
@@ -14,12 +14,12 @@ engine = create_engine(DATABASE_URL)
ToarDbSession = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# Dependency
-def get_engine():
+def get_engine(): # pragma: no cover
assert engine is not None
return engine
# Dependency
-def get_db():
+def get_db(): # pragma: no cover
try:
db = ToarDbSession()
yield db
--
GitLab
From 4343ae47b909b7df47b2b863d29f70a17c10c6f0 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 6 Dec 2024 15:29:54 +0000
Subject: [PATCH 17/36] deleted pragma instruction from else-case
---
toardb/test_base.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/toardb/test_base.py b/toardb/test_base.py
index e30e568..17125e8 100644
--- a/toardb/test_base.py
+++ b/toardb/test_base.py
@@ -51,7 +51,7 @@ async def override_dependency(request: Request):
"user_name": "Sabine Schröder",
"user_email": email,
"auth_user_id": db_user.id }
- else: # pragma: no cover
+ else:
# the user needs to be added to the database!
# (maybe users already have the credentials (in the AAI),
# but they also need a permanent auth_user_id related to the TOAR database)
--
GitLab
From 06d8b580ba0d057f31ebe796a703f059a45975eb Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 3 Jan 2025 16:10:47 +0000
Subject: [PATCH 18/36] corrected typo in HTAP Tier1 Region
---
...ar_controlled_vocabulary--0.7.6--0.7.7.sql | 27 +
.../toar_controlled_vocabulary--0.7.7.sql | 2224 +++++++++++++++++
.../toar_controlled_vocabulary.control | 4 +-
3 files changed, 2253 insertions(+), 2 deletions(-)
create mode 100644 extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql
create mode 100644 extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql
diff --git a/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql
new file mode 100644
index 0000000..258d6d9
--- /dev/null
+++ b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql
@@ -0,0 +1,27 @@
+--
+-- toardb/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql
+--
+-- [Step to install]
+--
+-- 1.
+--
+
+-- INSTALL VERSION: '0.7.7'
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE/ALTER EXTENSION toar_controlled_vocabulary" to load this file. \quit
+
+-- still to do:
+-- How to set convoc_schema to result from SELECT:
+-- SELECT table_schema INTO convoc_schema FROM information_schema.tables WHERE table_name='df_vocabulary';
+
+SET SCHEMA 'toar_convoc';
+
+-- Stationmeta
+-- ===========
+
+-- Station HTAP Regions (TIER1)
+-- The integer denoting the “tier1” region defined in the task force on hemispheric transport of air pollution (TFHTAP) coordinated model studies.
+
+UPDATE TR_vocabulary SET (enum_val, enum_str, enum_display_str) = ( 5, 'HTAPTier1SAS', '5 (SAS South Asia: India, Nepal, Pakistan, Afghanistan, Bangladesh, Sri Lanka)') WHERE enum_val=5;
+
diff --git a/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql
new file mode 100644
index 0000000..269164a
--- /dev/null
+++ b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql
@@ -0,0 +1,2224 @@
+--
+-- toardb/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql
+--
+-- [Step to install]
+--
+-- 1.
+--
+
+-- INSTALL VERSION: '0.7.7'
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION toar_controlled_vocabulary" to load this file. \quit
+
+-- Roles
+-- =====
+
+-- Role Codes
+
+CREATE TABLE IF NOT EXISTS RC_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT rc_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO RC_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'PointOfContact', 'point of contact'),
+ (1, 'PrincipalInvestigator', 'principal investigator'),
+ (2, 'Originator', 'originator'),
+ (3, 'Contributor', 'contributor'),
+ (4, 'Collaborator', 'collaborator'),
+ (5, 'ResourceProvider', 'resource provider'),
+ (6, 'Custodian', 'custodian'),
+ (7, 'Stakeholder', 'stakeholder'),
+ (8, 'RightsHolder', 'rights holder');
+
+-- Role Status
+
+CREATE TABLE IF NOT EXISTS RS_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT rs_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO RS_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Active', 'active'),
+ (1, 'Inactive', 'inactive'),
+ (2, 'Unknown', 'unknown');
+
+-- Annotation
+-- ==========
+
+-- Kind of Annotation
+
+CREATE TABLE IF NOT EXISTS AK_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT ak_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO AK_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'User', 'user comment'),
+ (1, 'Provider', 'provider comment'),
+ (2, 'Curator', 'curator comment');
+
+-- Contacts
+-- ========
+
+-- Kind of Organizations
+
+CREATE TABLE IF NOT EXISTS OK_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT ok_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO OK_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (1, 'Government', 'government'),
+ (2, 'Research', 'research'),
+ (3, 'University', 'university'),
+ (4, 'International', 'international'),
+ (5, 'NonProfit', 'non-profit'),
+ (6, 'Commercial', 'commercial'),
+ (7, 'Individual', 'individual'),
+ (8, 'Other', 'other');
+
+-- Changelogs
+-- ==========
+
+-- Type of Change
+
+CREATE TABLE IF NOT EXISTS CL_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT cl_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO CL_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Created', 'created'),
+ (1, 'SingleValue', 'single value correction in metadata'),
+ (2, 'Comprehensive', 'comprehensive metadata revision'),
+ (3, 'Typo', 'typographic correction of metadata'),
+ (4, 'UnspecifiedData', 'unspecified data value corrections'),
+ (5, 'Replaced', 'replaced data with a new version'),
+ (6, 'Flagging', 'data value flagging');
+
+
+-- Timeseries
+-- ==========
+
+-- Sampling Frequencies
+
+CREATE TABLE IF NOT EXISTS SF_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT sf_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO SF_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ ( 0, 'Hourly', 'hourly'),
+ ( 1, 'TenMinutes', '10-minutes'),
+ ( 2, 'FifteenMinutes', '15-minutes'),
+ ( 3, 'TwentyMinutes', '20-minutes'),
+ ( 4, 'ThirtyMinutes', '30-minutes'),
+ ( 5, 'ThreeHourly', '3-hourly'),
+ ( 6, 'SixHourly', '6-hourly'),
+ ( 7, 'Daily', 'daily'),
+ ( 8, 'Weekly', 'weekly'),
+ ( 9, 'Monthly', 'monthly'),
+ (10, 'Yearly', 'yearly'),
+ (11, 'Irregular', 'irregular data samples of constant length'),
+ (12, 'Irregular2', 'irregular data samples of varying length');
+
+-- Aggregation Types
+
+CREATE TABLE IF NOT EXISTS AT_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT at_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO AT_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Mean', 'mean'),
+ (1, 'MeanOf2', 'mean of two values'),
+ (2, 'MeanOfWeek', 'weekly mean'),
+ (3, 'MeanOf4Samples', 'mean out of 4 samples'),
+ (4, 'MeanOfMonth', 'monthly mean'),
+ (5, 'None', 'none'),
+ (6, 'Unknown', 'unknown');
+
+-- Data Origin Type
+
+CREATE TABLE IF NOT EXISTS OT_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT ot_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO OT_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Measurement', 'measurement'),
+ (1, 'Model', 'model');
+
+-- Data Origin
+
+CREATE TABLE IF NOT EXISTS DO_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT do_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO DO_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Instrument', 'instrument'),
+ (1, 'COSMOREA6', 'COSMO REA 6'),
+ (2, 'ERA5', 'ERA5');
+
+-- Stationmeta
+-- ===========
+
+-- climatic zones
+-- see: IPCC Climate Zone Map reported in Figure 3A.5.1 in Chapter 3 of Vol.4 of 2019 Refinement
+
+CREATE TABLE IF NOT EXISTS CZ_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT cz_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO CZ_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (-1, 'Undefined', 'undefined'),
+ ( 0, 'Unclassified', 'unclassified'),
+ ( 1, 'TropicalMontane', '1 (tropical montane)'),
+ ( 2, 'TropicalWet', '2 (tropical wet)'),
+ ( 3, 'TropicalMoist', '3 (tropical moist)'),
+ ( 4, 'TropicalDry', '4 (tropical dry)'),
+ ( 5, 'WarmTemperateMoist', '5 (warm temperate moist)'),
+ ( 6, 'WarmTemperateDry', '6 (warm temperate dry)'),
+ ( 7, 'CoolTemperateMoist', '7 (cool temperate moist)'),
+ ( 8, 'CoolTemperateDry', '8 (cool temperate dry)'),
+ ( 9, 'BorealMoist', '9 (boreal moist)'),
+ (10, 'BorealDry', '10 (boreal dry)'),
+ (11, 'PolarMoist', '11 (polar moist)'),
+ (12, 'PolarDry', '12 (polar dry)');
+
+-- Station Coordinate Validity
+
+CREATE TABLE IF NOT EXISTS CV_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT cv_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO CV_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'NotChecked', 'not checked'),
+ (1, 'Verified', 'verified'),
+ (2, 'Plausible', 'plausible'),
+ (3, 'Doubtful', 'doubtful'),
+ (4, 'Unverifyable', 'not verifyable');
+
+-- Station Types
+
+CREATE TABLE IF NOT EXISTS ST_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT st_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO ST_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Unknown', 'unknown'),
+ (1, 'Background', 'background'),
+ (2, 'Traffic', 'traffic'),
+ (3, 'Industrial', 'industrial');
+
+-- Station Types Of Area
+
+CREATE TABLE IF NOT EXISTS TA_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT ta_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO TA_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Unknown', 'unknown'),
+ (1, 'Urban', 'urban'),
+ (2, 'Suburban', 'suburban'),
+ (3, 'Rural', 'rural');
+
+-- Station TOAR Categories
+
+CREATE TABLE IF NOT EXISTS TC_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT tc_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO TC_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (-1, 'Unknown', 'unknown'),
+ ( 0, 'Unclassified', 'unclassified'),
+ ( 1, 'RuralLowElevation', 'rural low elevation'),
+ ( 2, 'RuralHighElevation', 'rural high elevation'),
+ ( 3, 'Urban', 'urban');
+
+-- Station HTAP Regions (TIER1)
+-- The integer denoting the “tier1” region defined in the task force on hemispheric transport of air pollution (TFHTAP) coordinated model studies.
+
+CREATE TABLE IF NOT EXISTS TR_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT tr_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO TR_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (-1, 'HTAPTier1Undefined', '-1 (undefined)'),
+ ( 1, 'HTAPTier1World', '1 (World)'),
+ ( 2, 'HTAPTier1OCN', '2 (OCN Non-arctic/Antarctic Ocean)'),
+ ( 3, 'HTAPTier1NAM', '3 (NAM US+Canada (upto 66 N; polar circle))'),
+ ( 4, 'HTAPTier1EUR', '4 (EUR Western + Eastern EU+Turkey (upto 66 N polar circle))'),
+ ( 5, 'HTAPTier1SAS', '5 (SAS South Asia: India, Nepal, Pakistan, Afghanistan, Bangladesh, Sri Lanka)'),
+ ( 6, 'HTAPTier1EAS', '6 (EAS East Asia: China, Korea, Japan)'),
+ ( 7, 'HTAPTier1SEA', '7 (SEA South East Asia)'),
+ ( 8, 'HTAPTier1PAN', '8 (PAN Pacific, Australia+ New Zealand)'),
+ ( 9, 'HTAPTier1NAF', '9 (NAF Northern Africa+Sahara+Sahel)'),
+ (10, 'HTAPTier1SAF', '10 (SAF Sub Saharan/sub Sahel Africa)'),
+ (11, 'HTAPTier1MDE', '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)'),
+ (12, 'HTAPTier1MCA', '12 (MCA Mexico, Central America, Caribbean, Guyanas, Venezuela, Columbia)'),
+ (13, 'HTAPTier1SAM', '13 (SAM S. America)'),
+ (14, 'HTAPTier1RBU', '14 (RBU Russia, Belarussia, Ukraine)'),
+ (15, 'HTAPTier1CAS', '15 (CAS Central Asia)'),
+ (16, 'HTAPTier1NPO', '16 (NPO Arctic Circle (North of 66 N) + Greenland)'),
+ (17, 'HTAPTier1SPO', '17 (SPO Antarctic)');
+
+-- Station Landcover Types
+-- see: http://maps.elie.ucl.ac.be/CCI/viewer/download/ESACCI-LC-Ph2-PUGv2_2.0.pdf
+
+CREATE TABLE IF NOT EXISTS LC_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT dl_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO LC_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ ( -1, 'Undefined', '-1 (undefined)'),
+ ( 0, 'NoData', '0 (No Data)'),
+ ( 10, 'CroplandRainfed', '10 (Cropland, rainfed)'),
+ ( 11, 'CroplandRainfedHerbaceousCover', '11 (Cropland, rainfed, herbaceous cover)'),
+ ( 12, 'CroplandRainfedTreeOrShrubCover', '12 (Cropland, rainfed, tree or shrub cover)'),
+ ( 20, 'CroplandIrrigated', '20 (Cropland, irrigated or post-flooding)'),
+ ( 30, 'MosaicCropland', '30 (Mosaic cropland (>50%) / natural vegetation (tree, shrub, herbaceous cover) (<50%))'),
+ ( 40, 'MosaicNaturalVegetation', '40 (Mosaic natural vegetation (tree, shrub, herbaceous cover) (>50%) / cropland (<50%))'),
+ ( 50, 'TreeBroadleavedEvergreenClosedToOpen', '50 (Tree cover, broadleaved, evergreen, closed to open (>15%))'),
+ ( 60, 'TreeBroadleavedDeciduousClosedToOpen', '60 (Tree cover, broadleaved, deciduous, closed to open (>15%))'),
+ ( 61, 'TreeBroadleavedDeciduousClosed', '61 (Tree cover, broadleaved, deciduous, closed (>40%))'),
+ ( 62, 'TreeBroadleavedDeciduousOpen', '62 (Tree cover, broadleaved, deciduous, open (15-40%))'),
+ ( 70, 'TreeNeedleleavedEvergreenClosedToOpen', '70 (Tree cover, needleleaved, evergreen, closed to open (>15%))'),
+ ( 71, 'TreeNeedleleavedEvergreenClosed', '71 (Tree cover, needleleaved, evergreen, closed (>40%))'),
+ ( 72, 'TreeNeedleleavedEvergreenOpen', '72 (Tree cover, needleleaved, evergreen, open (15-40%))'),
+ ( 80, 'TreeNeedleleavedDeciduousClosedToOpen', '80 (Tree cover, needleleaved, deciduous, closed to open (>15%))'),
+ ( 81, 'TreeNedleleavedDeciduousClosed', '81 (Tree cover, needleleaved, deciduous, closed (>40%))'),
+ ( 82, 'TreeNeedleleavedDeciduousOpen', '82 (Tree cover, needleleaved, deciduous, open (15-40%))'),
+ ( 90, 'TreeMixed', '90 (Tree cover, mixed leaf type (broadleaved and needleleaved))'),
+ (100, 'MosaicTreeAndShrub', '100 (Mosaic tree and shrub (>50%) / herbaceous cover (<50%))'),
+ (110, 'MosaicHerbaceous', '110 (Mosaic herbaceous cover (>50%) / tree and shrub (<50%))'),
+ (120, 'Shrubland', '120 (Shrubland)'),
+ (121, 'ShrublandEvergreen', '121 (Evergreen shrubland)'),
+ (122, 'ShrublandDeciduous', '122 (Deciduous shrubland)'),
+ (130, 'Grassland', '130 (Grassland)'),
+ (140, 'LichensAndMosses', '140 (Lichens and mosses)'),
+ (150, 'SparseVegetation', '150 (Sparse vegetation (tree, shrub, herbaceous cover) (<15%))'),
+ (151, 'SparseTree', '151 (Sparse tree (<15%))'),
+ (152, 'SparseShrub', '152 (Sparse shrub (<15%))'),
+ (153, 'SparseHerbaceous', '153 (Sparse herbaceous cover (<15%))'),
+ (160, 'TreeCoverFloodedFreshOrBrakishWater', '160 (Tree cover, flooded, fresh or brakish water)'),
+ (170, 'TreeCoverFloodedSalineWater', '170 (Tree cover, flooded, saline water)'),
+ (180, 'ShrubOrHerbaceousCoverFlooded', '180 (Shrub or herbaceous cover, flooded, fresh/saline/brakish water)'),
+ (190, 'Urban', '190 (Urban areas)'),
+ (200, 'BareAreas', '200 (Bare areas)'),
+ (201, 'BareAreasConsolidated', '201 (Consolidated bare areas)'),
+ (202, 'BareAreasUnconsolidated', '202 (Unconsolidated bare areas)'),
+ (210, 'Water', '210 (Water bodies)'),
+ (220, 'SnowAndIce', '220 (Permanent snow and ice)');
+
+-- Station Eco Regions
+-- see: https://ecoregions2017.appspot.com/
+
+CREATE TABLE IF NOT EXISTS ER_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT er_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO ER_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ ( -1, 'Undefined', '-1 (undefined)'),
+ ( 0, 'RockandIce', '0 (Rock and Ice)'),
+ ( 1, 'AlbertineRiftmontaneforests', '1 (Albertine Rift montane forests)'),
+ ( 2, 'CameroonHighlandsforests', '2 (Cameroon Highlands forests)'),
+ ( 3, 'CentralCongolianlowlandforests', '3 (Central Congolian lowland forests)'),
+ ( 4, 'Comorosforests', '4 (Comoros forests)'),
+ ( 5, 'Congoliancoastalforests', '5 (Congolian coastal forests)'),
+ ( 6, 'CrossNigertransitionforests', '6 (Cross-Niger transition forests)'),
+ ( 7, 'CrossSanagaBiokocoastalforests', '7 (Cross-Sanaga-Bioko coastal forests)'),
+ ( 8, 'EastAfricanmontaneforests', '8 (East African montane forests)'),
+ ( 9, 'EasternArcforests', '9 (Eastern Arc forests)'),
+ ( 10, 'EasternCongolianswampforests', '10 (Eastern Congolian swamp forests)'),
+ ( 11, 'EasternGuineanforests', '11 (Eastern Guinean forests)'),
+ ( 12, 'Ethiopianmontaneforests', '12 (Ethiopian montane forests)'),
+ ( 13, 'GraniticSeychellesforests', '13 (Granitic Seychelles forests)'),
+ ( 14, 'Guineanmontaneforests', '14 (Guinean montane forests)'),
+ ( 15, 'KnysnaAmatolemontaneforests', '15 (Knysna-Amatole montane forests)'),
+ ( 16, 'KwazuluNatalCapecoastalforests', '16 (Kwazulu Natal-Cape coastal forests)'),
+ ( 17, 'Madagascarhumidforests', '17 (Madagascar humid forests)'),
+ ( 18, 'Madagascarsubhumidforests', '18 (Madagascar subhumid forests)'),
+ ( 19, 'Maputalandcoastalforestsandwoodlands', '19 (Maputaland coastal forests and woodlands)'),
+ ( 20, 'Mascareneforests', '20 (Mascarene forests)'),
+ ( 21, 'MountCameroonandBiokomontaneforests', '21 (Mount Cameroon and Bioko montane forests)'),
+ ( 22, 'NigerDeltaswampforests', '22 (Niger Delta swamp forests)'),
+ ( 23, 'Nigerianlowlandforests', '23 (Nigerian lowland forests)'),
+ ( 24, 'NortheastCongolianlowlandforests', '24 (Northeast Congolian lowland forests)'),
+ ( 25, 'NorthernSwahilicoastalforests', '25 (Northern Swahili coastal forests)'),
+ ( 26, 'NorthwestCongolianlowlandforests', '26 (Northwest Congolian lowland forests)'),
+ ( 27, 'SaoTomePrincipeandAnnobonforests', '27 (São Tomé, Príncipe, and Annobón forests)'),
+ ( 28, 'SouthernSwahilicoastalforestsandwoodlands', '28 (Southern Swahili coastal forests and woodlands)'),
+ ( 29, 'WesternCongolianswampforests', '29 (Western Congolian swamp forests)'),
+ ( 30, 'WesternGuineanlowlandforests', '30 (Western Guinean lowland forests)'),
+ ( 31, 'CapeVerdeIslandsdryforests', '31 (Cape Verde Islands dry forests)'),
+ ( 32, 'Madagascardrydeciduousforests', '32 (Madagascar dry deciduous forests)'),
+ ( 33, 'Zambezianevergreendryforests', '33 (Zambezian evergreen dry forests)'),
+ ( 34, 'Angolanmopanewoodlands', '34 (Angolan mopane woodlands)'),
+ ( 35, 'Angolanscarpsavannaandwoodlands', '35 (Angolan scarp savanna and woodlands)'),
+ ( 36, 'Angolanwetmiombowoodlands', '36 (Angolan wet miombo woodlands)'),
+ ( 37, 'Ascensionscrubandgrasslands', '37 (Ascension scrub and grasslands)'),
+ ( 38, 'Centralbushveld', '38 (Central bushveld)'),
+ ( 39, 'CentralZambezianwetmiombowoodlands', '39 (Central Zambezian wet miombo woodlands)'),
+ ( 40, 'DrakensbergEscarpmentsavannaandthicket', '40 (Drakensberg Escarpment savanna and thicket)'),
+ ( 41, 'Drakensberggrasslands', '41 (Drakensberg grasslands)'),
+ ( 42, 'Drymiombowoodlands', '42 (Dry miombo woodlands)'),
+ ( 43, 'EastSudaniansavanna', '43 (East Sudanian savanna)'),
+ ( 44, 'Guineanforestsavanna', '44 (Guinean forest-savanna)'),
+ ( 45, 'HornofAfricaxericbushlands', '45 (Horn of Africa xeric bushlands)'),
+ ( 46, 'ItigiSumbuthicket', '46 (Itigi-Sumbu thicket)'),
+ ( 47, 'KalahariAcaciawoodlands', '47 (Kalahari Acacia woodlands)'),
+ ( 48, 'Limpopolowveld', '48 (Limpopo lowveld)'),
+ ( 49, 'MandaraPlateauwoodlands', '49 (Mandara Plateau woodlands)'),
+ ( 50, 'Masaixericgrasslandsandshrublands', '50 (Masai xeric grasslands and shrublands)'),
+ ( 51, 'NorthernAcaciaCommiphorabushlandsandthickets', '51 (Northern Acacia-Commiphora bushlands and thickets)'),
+ ( 52, 'NorthernCongolianForestSavanna', '52 (Northern Congolian Forest-Savanna)'),
+ ( 53, 'SahelianAcaciasavanna', '53 (Sahelian Acacia savanna)'),
+ ( 54, 'Serengetivolcanicgrasslands', '54 (Serengeti volcanic grasslands)'),
+ ( 55, 'SomaliAcaciaCommiphorabushlandsandthickets', '55 (Somali Acacia-Commiphora bushlands and thickets)'),
+ ( 56, 'SouthArabianfogwoodlandsshrublandsanddune', '56 (South Arabian fog woodlands, shrublands, and dune)'),
+ ( 57, 'SouthernAcaciaCommiphorabushlandsandthickets', '57 (Southern Acacia-Commiphora bushlands and thickets)'),
+ ( 58, 'SouthernCongolianforestsavanna', '58 (Southern Congolian forest-savanna)'),
+ ( 59, 'SouthwestArabianmontanewoodlandsandgrasslands', '59 (Southwest Arabian montane woodlands and grasslands)'),
+ ( 60, 'StHelenascrubandwoodlands', '60 (St. Helena scrub and woodlands)'),
+ ( 61, 'VictoriaBasinforestsavanna', '61 (Victoria Basin forest-savanna)'),
+ ( 62, 'WestSudaniansavanna', '62 (West Sudanian savanna)'),
+ ( 63, 'WesternCongolianforestsavanna', '63 (Western Congolian forest-savanna)'),
+ ( 64, 'ZambezianBaikiaeawoodlands', '64 (Zambezian Baikiaea woodlands)'),
+ ( 65, 'Zambezianmopanewoodlands', '65 (Zambezian mopane woodlands)'),
+ ( 66, 'ZambezianLimpopomixedwoodlands', '66 (Zambezian-Limpopo mixed woodlands)'),
+ ( 67, 'AmsterdamSaintPaulIslandstemperategrasslands', '67 (Amsterdam-Saint Paul Islands temperate grasslands)'),
+ ( 68, 'TristanDaCunhaGoughIslandsshrubandgrasslands', '68 (Tristan Da Cunha-Gough Islands shrub and grasslands)'),
+ ( 69, 'EastAfricanhalophytics', '69 (East African halophytics)'),
+ ( 70, 'EtoshaPanhalophytics', '70 (Etosha Pan halophytics)'),
+ ( 71, 'InnerNigerDeltafloodedsavanna', '71 (Inner Niger Delta flooded savanna)'),
+ ( 72, 'LakeChadfloodedsavanna', '72 (Lake Chad flooded savanna)'),
+ ( 73, 'Makgadikgadihalophytics', '73 (Makgadikgadi halophytics)'),
+ ( 74, 'Suddfloodedgrasslands', '74 (Sudd flooded grasslands)'),
+ ( 75, 'Zambeziancoastalfloodedsavanna', '75 (Zambezian coastal flooded savanna)'),
+ ( 76, 'Zambezianfloodedgrasslands', '76 (Zambezian flooded grasslands)'),
+ ( 77, 'Angolanmontaneforestgrassland', '77 (Angolan montane forest-grassland)'),
+ ( 78, 'EastAfricanmontanemoorlands', '78 (East African montane moorlands)'),
+ ( 79, 'Ethiopianmontanegrasslandsandwoodlands', '79 (Ethiopian montane grasslands and woodlands)'),
+ ( 80, 'Ethiopianmontanemoorlands', '80 (Ethiopian montane moorlands)'),
+ ( 81, 'Highveldgrasslands', '81 (Highveld grasslands)'),
+ ( 82, 'JosPlateauforestgrassland', '82 (Jos Plateau forest-grassland)'),
+ ( 83, 'Madagascarericoidthickets', '83 (Madagascar ericoid thickets)'),
+ ( 84, 'MulanjeMontaneforestgrassland', '84 (Mulanje Montane forest-grassland)'),
+ ( 85, 'NyangaChimanimaniMontaneforestgrassland', '85 (Nyanga-Chimanimani Montane forest-grassland)'),
+ ( 86, 'RwenzoriVirungamontanemoorlands', '86 (Rwenzori-Virunga montane moorlands)'),
+ ( 87, 'SouthernRiftMontaneforestgrassland', '87 (Southern Rift Montane forest-grassland)'),
+ ( 88, 'Albanythickets', '88 (Albany thickets)'),
+ ( 89, 'Fynbosshrubland', '89 (Fynbos shrubland)'),
+ ( 90, 'Renosterveldshrubland', '90 (Renosterveld shrubland)'),
+ ( 91, 'AldabraIslandxericscrub', '91 (Aldabra Island xeric scrub)'),
+ ( 92, 'Djiboutixericshrublands', '92 (Djibouti xeric shrublands)'),
+ ( 93, 'Eritreancoastaldesert', '93 (Eritrean coastal desert)'),
+ ( 94, 'GariepKaroo', '94 (Gariep Karoo)'),
+ ( 95, 'Hobyograsslandsandshrublands', '95 (Hobyo grasslands and shrublands)'),
+ ( 96, 'IleEuropaandBassasdaIndiaxericscrub', '96 (Ile Europa and Bassas da India xeric scrub)'),
+ ( 97, 'Kalaharixericsavanna', '97 (Kalahari xeric savanna)'),
+ ( 98, 'Kaokovelddesert', '98 (Kaokoveld desert)'),
+ ( 99, 'Madagascarspinythickets', '99 (Madagascar spiny thickets)'),
+ (100, 'Madagascarsucculentwoodlands', '100 (Madagascar succulent woodlands)'),
+ (101, 'NamaKarooshrublands', '101 (Nama Karoo shrublands)'),
+ (102, 'NamaqualandRichtersveldsteppe', '102 (Namaqualand-Richtersveld steppe)'),
+ (103, 'NamibDesert', '103 (Namib Desert)'),
+ (104, 'Namibiansavannawoodlands', '104 (Namibian savanna woodlands)'),
+ (105, 'SocotraIslandxericshrublands', '105 (Socotra Island xeric shrublands)'),
+ (106, 'Somalimontanexericwoodlands', '106 (Somali montane xeric woodlands)'),
+ (107, 'SouthwestArabiancoastalxericshrublands', '107 (Southwest Arabian coastal xeric shrublands)'),
+ (108, 'SouthwestArabianEscarpmentshrublandsandwoodlands', '108 (Southwest Arabian Escarpment shrublands and woodlands)'),
+ (109, 'SouthwestArabianhighlandxericscrub', '109 (Southwest Arabian highland xeric scrub)'),
+ (110, 'SucculentKarooxericshrublands', '110 (Succulent Karoo xeric shrublands)'),
+ (111, 'CentralAfricanmangroves', '111 (Central African mangroves)'),
+ (112, 'EastAfricanmangroves', '112 (East African mangroves)'),
+ (113, 'Guineanmangroves', '113 (Guinean mangroves)'),
+ (114, 'Madagascarmangroves', '114 (Madagascar mangroves)'),
+ (115, 'RedSeamangroves', '115 (Red Sea mangroves)'),
+ (116, 'SouthernAfricamangroves', '116 (Southern Africa mangroves)'),
+ (117, 'AdelieLandtundra', '117 (Adelie Land tundra)'),
+ (118, 'CentralSouthAntarcticPeninsulatundra', '118 (Central South Antarctic Peninsula tundra)'),
+ (119, 'DronningMaudLandtundra', '119 (Dronning Maud Land tundra)'),
+ (120, 'EastAntarctictundra', '120 (East Antarctic tundra)'),
+ (121, 'EllsworthLandtundra', '121 (Ellsworth Land tundra)'),
+ (122, 'EllsworthMountainstundra', '122 (Ellsworth Mountains tundra)'),
+ (123, 'EnderbyLandtundra', '123 (Enderby Land tundra)'),
+ (124, 'MarieByrdLandtundra', '124 (Marie Byrd Land tundra)'),
+ (125, 'NorthVictoriaLandtundra', '125 (North Victoria Land tundra)'),
+ (126, 'NortheastAntarcticPeninsulatundra', '126 (Northeast Antarctic Peninsula tundra)'),
+ (127, 'NorthwestAntarcticPeninsulatundra', '127 (Northwest Antarctic Peninsula tundra)'),
+ (128, 'PrinceCharlesMountainstundra', '128 (Prince Charles Mountains tundra)'),
+ (129, 'ScotiaSeaIslandstundra', '129 (Scotia Sea Islands tundra)'),
+ (130, 'SouthAntarcticPeninsulatundra', '130 (South Antarctic Peninsula tundra)'),
+ (131, 'SouthOrkneyIslandstundra', '131 (South Orkney Islands tundra)'),
+ (132, 'SouthVictoriaLandtundra', '132 (South Victoria Land tundra)'),
+ (133, 'SouthernIndianOceanIslandstundra', '133 (Southern Indian Ocean Islands tundra)'),
+ (134, 'TransantarcticMountainstundra', '134 (Transantarctic Mountains tundra)'),
+ (135, 'AdmiraltyIslandslowlandrainforests', '135 (Admiralty Islands lowland rain forests)'),
+ (136, 'BandaSeaIslandsmoistdeciduousforests', '136 (Banda Sea Islands moist deciduous forests)'),
+ (137, 'BiakNumfoorrainforests', '137 (Biak-Numfoor rain forests)'),
+ (138, 'Bururainforests', '138 (Buru rain forests)'),
+ (139, 'CentralRangePapuanmontanerainforests', '139 (Central Range Papuan montane rain forests)'),
+ (140, 'Halmaherarainforests', '140 (Halmahera rain forests)'),
+ (141, 'HuonPeninsulamontanerainforests', '141 (Huon Peninsula montane rain forests)'),
+ (142, 'LordHoweIslandsubtropicalforests', '142 (Lord Howe Island subtropical forests)'),
+ (143, 'LouisiadeArchipelagorainforests', '143 (Louisiade Archipelago rain forests)'),
+ (144, 'NewBritainNewIrelandlowlandrainforests', '144 (New Britain-New Ireland lowland rain forests)'),
+ (145, 'NewBritainNewIrelandmontanerainforests', '145 (New Britain-New Ireland montane rain forests)'),
+ (146, 'NewCaledoniarainforests', '146 (New Caledonia rain forests)'),
+ (147, 'NorfolkIslandsubtropicalforests', '147 (Norfolk Island subtropical forests)'),
+ (148, 'NorthernNewGuinealowlandrainandfreshwaterswampforests', '148 (Northern New Guinea lowland rain and freshwater swamp forests)'),
+ (149, 'NorthernNewGuineamontanerainforests', '149 (Northern New Guinea montane rain forests)'),
+ (150, 'Queenslandtropicalrainforests', '150 (Queensland tropical rain forests)'),
+ (151, 'Seramrainforests', '151 (Seram rain forests)'),
+ (152, 'SolomonIslandsrainforests', '152 (Solomon Islands rain forests)'),
+ (153, 'SoutheastPapuanrainforests', '153 (Southeast Papuan rain forests)'),
+ (154, 'SouthernNewGuineafreshwaterswampforests', '154 (Southern New Guinea freshwater swamp forests)'),
+ (155, 'SouthernNewGuinealowlandrainforests', '155 (Southern New Guinea lowland rain forests)'),
+ (156, 'Sulawesilowlandrainforests', '156 (Sulawesi lowland rain forests)'),
+ (157, 'Sulawesimontanerainforests', '157 (Sulawesi montane rain forests)'),
+ (158, 'TrobriandIslandsrainforests', '158 (Trobriand Islands rain forests)'),
+ (159, 'Vanuaturainforests', '159 (Vanuatu rain forests)'),
+ (160, 'Vogelkopmontanerainforests', '160 (Vogelkop montane rain forests)'),
+ (161, 'VogelkopArulowlandrainforests', '161 (Vogelkop-Aru lowland rain forests)'),
+ (162, 'Yapenrainforests', '162 (Yapen rain forests)'),
+ (163, 'LesserSundasdeciduousforests', '163 (Lesser Sundas deciduous forests)'),
+ (164, 'NewCaledoniadryforests', '164 (New Caledonia dry forests)'),
+ (165, 'Sumbadeciduousforests', '165 (Sumba deciduous forests)'),
+ (166, 'TimorandWetardeciduousforests', '166 (Timor and Wetar deciduous forests)'),
+ (167, 'ChathamIslandtemperateforests', '167 (Chatham Island temperate forests)'),
+ (168, 'EasternAustraliantemperateforests', '168 (Eastern Australian temperate forests)'),
+ (169, 'Fiordlandtemperateforests', '169 (Fiordland temperate forests)'),
+ (170, 'NelsonCoasttemperateforests', '170 (Nelson Coast temperate forests)'),
+ (171, 'NewZealandNorthIslandtemperateforests', '171 (New Zealand North Island temperate forests)'),
+ (172, 'NewZealandSouthIslandtemperateforests', '172 (New Zealand South Island temperate forests)'),
+ (173, 'Northlandtemperatekauriforests', '173 (Northland temperate kauri forests)'),
+ (174, 'RakiuraIslandtemperateforests', '174 (Rakiura Island temperate forests)'),
+ (175, 'Richmondtemperateforests', '175 (Richmond temperate forests)'),
+ (176, 'SoutheastAustraliatemperateforests', '176 (Southeast Australia temperate forests)'),
+ (177, 'TasmanianCentralHighlandforests', '177 (Tasmanian Central Highland forests)'),
+ (178, 'Tasmaniantemperateforests', '178 (Tasmanian temperate forests)'),
+ (179, 'Tasmaniantemperaterainforests', '179 (Tasmanian temperate rain forests)'),
+ (180, 'Westlandtemperateforests', '180 (Westland temperate forests)'),
+ (181, 'ArnhemLandtropicalsavanna', '181 (Arnhem Land tropical savanna)'),
+ (182, 'Brigalowtropicalsavanna', '182 (Brigalow tropical savanna)'),
+ (183, 'CapeYorkPeninsulatropicalsavanna', '183 (Cape York Peninsula tropical savanna)'),
+ (184, 'Carpentariatropicalsavanna', '184 (Carpentaria tropical savanna)'),
+ (185, 'Einasleighuplandsavanna', '185 (Einasleigh upland savanna)'),
+ (186, 'Kimberlytropicalsavanna', '186 (Kimberly tropical savanna)'),
+ (187, 'MitchellGrassDowns', '187 (Mitchell Grass Downs)'),
+ (188, 'TransFlysavannaandgrasslands', '188 (Trans Fly savanna and grasslands)'),
+ (189, 'VictoriaPlainstropicalsavanna', '189 (Victoria Plains tropical savanna)'),
+ (190, 'CanterburyOtagotussockgrasslands', '190 (Canterbury-Otago tussock grasslands)'),
+ (191, 'EasternAustraliamulgashrublands', '191 (Eastern Australia mulga shrublands)'),
+ (192, 'SoutheastAustraliatemperatesavanna', '192 (Southeast Australia temperate savanna)'),
+ (193, 'AustralianAlpsmontanegrasslands', '193 (Australian Alps montane grasslands)'),
+ (194, 'NewZealandSouthIslandmontanegrasslands', '194 (New Zealand South Island montane grasslands)'),
+ (195, 'PapuanCentralRangesubalpinegrasslands', '195 (Papuan Central Range sub-alpine grasslands)'),
+ (196, 'AntipodesSubantarcticIslandstundra', '196 (Antipodes Subantarctic Islands tundra)'),
+ (197, 'Coolgardiewoodlands', '197 (Coolgardie woodlands)'),
+ (198, 'Esperancemallee', '198 (Esperance mallee)'),
+ (199, 'EyreandYorkmallee', '199 (Eyre and York mallee)'),
+ (200, 'FlindersLoftymontanewoodlands', '200 (Flinders-Lofty montane woodlands)'),
+ (201, 'Hamptonmalleeandwoodlands', '201 (Hampton mallee and woodlands)'),
+ (202, 'JarrahKarriforestandshrublands', '202 (Jarrah-Karri forest and shrublands)'),
+ (203, 'MurrayDarlingwoodlandsandmallee', '203 (Murray-Darling woodlands and mallee)'),
+ (204, 'Naracoortewoodlands', '204 (Naracoorte woodlands)'),
+ (205, 'SouthwestAustraliasavanna', '205 (Southwest Australia savanna)'),
+ (206, 'SouthwestAustraliawoodlands', '206 (Southwest Australia woodlands)'),
+ (207, 'Carnarvonxericshrublands', '207 (Carnarvon xeric shrublands)'),
+ (208, 'CentralRangesxericscrub', '208 (Central Ranges xeric scrub)'),
+ (209, 'Gibsondesert', '209 (Gibson desert)'),
+ (210, 'GreatSandyTanamidesert', '210 (Great Sandy-Tanami desert)'),
+ (211, 'GreatVictoriadesert', '211 (Great Victoria desert)'),
+ (212, 'NullarborPlainsxericshrublands', '212 (Nullarbor Plains xeric shrublands)'),
+ (213, 'Pilbarashrublands', '213 (Pilbara shrublands)'),
+ (214, 'Simpsondesert', '214 (Simpson desert)'),
+ (215, 'TirariSturtstonydesert', '215 (Tirari-Sturt stony desert)'),
+ (216, 'WesternAustralianMulgashrublands', '216 (Western Australian Mulga shrublands)'),
+ (217, 'NewGuineamangroves', '217 (New Guinea mangroves)'),
+ (218, 'AndamanIslandsrainforests', '218 (Andaman Islands rain forests)'),
+ (219, 'Borneolowlandrainforests', '219 (Borneo lowland rain forests)'),
+ (220, 'Borneomontanerainforests', '220 (Borneo montane rain forests)'),
+ (221, 'Borneopeatswampforests', '221 (Borneo peat swamp forests)'),
+ (222, 'BrahmaputraValleysemievergreenforests', '222 (Brahmaputra Valley semi-evergreen forests)'),
+ (223, 'CardamomMountainsrainforests', '223 (Cardamom Mountains rain forests)'),
+ (224, 'ChaoPhrayafreshwaterswampforests', '224 (Chao Phraya freshwater swamp forests)'),
+ (225, 'ChaoPhrayalowlandmoistdeciduousforests', '225 (Chao Phraya lowland moist deciduous forests)'),
+ (226, 'ChinHillsArakanYomamontaneforests', '226 (Chin Hills-Arakan Yoma montane forests)'),
+ (227, 'ChristmasandCocosIslandstropicalforests', '227 (Christmas and Cocos Islands tropical forests)'),
+ (228, 'EastDeccanmoistdeciduousforests', '228 (East Deccan moist deciduous forests)'),
+ (229, 'EasternJavaBalimontanerainforests', '229 (Eastern Java-Bali montane rain forests)'),
+ (230, 'EasternJavaBalirainforests', '230 (Eastern Java-Bali rain forests)'),
+ (231, 'GreaterNegrosPanayrainforests', '231 (Greater Negros-Panay rain forests)'),
+ (232, 'HainanIslandmonsoonrainforests', '232 (Hainan Island monsoon rain forests)'),
+ (233, 'Himalayansubtropicalbroadleafforests', '233 (Himalayan subtropical broadleaf forests)'),
+ (234, 'Irrawaddyfreshwaterswampforests', '234 (Irrawaddy freshwater swamp forests)'),
+ (235, 'Irrawaddymoistdeciduousforests', '235 (Irrawaddy moist deciduous forests)'),
+ (236, 'JianNansubtropicalevergreenforests', '236 (Jian Nan subtropical evergreen forests)'),
+ (237, 'KayahKarenmontanerainforests', '237 (Kayah-Karen montane rain forests)'),
+ (238, 'LowerGangeticPlainsmoistdeciduousforests', '238 (Lower Gangetic Plains moist deciduous forests)'),
+ (239, 'LuangPrabangmontanerainforests', '239 (Luang Prabang montane rain forests)'),
+ (240, 'Luzonmontanerainforests', '240 (Luzon montane rain forests)'),
+ (241, 'Luzonrainforests', '241 (Luzon rain forests)'),
+ (242, 'MalabarCoastmoistforests', '242 (Malabar Coast moist forests)'),
+ (243, 'MaldivesLakshadweepChagosArchipelagotropicalmoistforests', '243 (Maldives-Lakshadweep-Chagos Archipelago tropical moist forests)'),
+ (244, 'Meghalayasubtropicalforests', '244 (Meghalaya subtropical forests)'),
+ (245, 'MentawaiIslandsrainforests', '245 (Mentawai Islands rain forests)'),
+ (246, 'Mindanaomontanerainforests', '246 (Mindanao montane rain forests)'),
+ (247, 'MindanaoEasternVisayasrainforests', '247 (Mindanao-Eastern Visayas rain forests)'),
+ (248, 'Mindororainforests', '248 (Mindoro rain forests)'),
+ (249, 'MizoramManipurKachinrainforests', '249 (Mizoram-Manipur-Kachin rain forests)'),
+ (250, 'Myanmarcoastalrainforests', '250 (Myanmar coastal rain forests)'),
+ (251, 'NanseiIslandssubtropicalevergreenforests', '251 (Nansei Islands subtropical evergreen forests)'),
+ (252, 'NicobarIslandsrainforests', '252 (Nicobar Islands rain forests)'),
+ (253, 'NorthWesternGhatsmoistdeciduousforests', '253 (North Western Ghats moist deciduous forests)'),
+ (254, 'NorthWesternGhatsmontanerainforests', '254 (North Western Ghats montane rain forests)'),
+ (255, 'NorthernAnnamitesrainforests', '255 (Northern Annamites rain forests)'),
+ (256, 'NorthernIndochinasubtropicalforests', '256 (Northern Indochina subtropical forests)'),
+ (257, 'NorthernKhoratPlateaumoistdeciduousforests', '257 (Northern Khorat Plateau moist deciduous forests)'),
+ (258, 'NorthernThailandLaosmoistdeciduousforests', '258 (Northern Thailand-Laos moist deciduous forests)'),
+ (259, 'NorthernTrianglesubtropicalforests', '259 (Northern Triangle subtropical forests)'),
+ (260, 'NorthernVietnamlowlandrainforests', '260 (Northern Vietnam lowland rain forests)'),
+ (261, 'Orissasemievergreenforests', '261 (Orissa semi-evergreen forests)'),
+ (262, 'Palawanrainforests', '262 (Palawan rain forests)'),
+ (263, 'PeninsularMalaysianmontanerainforests', '263 (Peninsular Malaysian montane rain forests)'),
+ (264, 'PeninsularMalaysianpeatswampforests', '264 (Peninsular Malaysian peat swamp forests)'),
+ (265, 'PeninsularMalaysianrainforests', '265 (Peninsular Malaysian rain forests)'),
+ (266, 'RedRiverfreshwaterswampforests', '266 (Red River freshwater swamp forests)'),
+ (267, 'SouthChinaSeaIslands', '267 (South China Sea Islands)'),
+ (268, 'SouthChinaVietnamsubtropicalevergreenforests', '268 (South China-Vietnam subtropical evergreen forests)'),
+ (269, 'SouthTaiwanmonsoonrainforests', '269 (South Taiwan monsoon rain forests)'),
+ (270, 'SouthWesternGhatsmoistdeciduousforests', '270 (South Western Ghats moist deciduous forests)'),
+ (271, 'SouthWesternGhatsmontanerainforests', '271 (South Western Ghats montane rain forests)'),
+ (272, 'SouthernAnnamitesmontanerainforests', '272 (Southern Annamites montane rain forests)'),
+ (273, 'SouthwestBorneofreshwaterswampforests', '273 (Southwest Borneo freshwater swamp forests)'),
+ (274, 'SriLankalowlandrainforests', '274 (Sri Lanka lowland rain forests)'),
+ (275, 'SriLankamontanerainforests', '275 (Sri Lanka montane rain forests)'),
+ (276, 'SuluArchipelagorainforests', '276 (Sulu Archipelago rain forests)'),
+ (277, 'Sumatranfreshwaterswampforests', '277 (Sumatran freshwater swamp forests)'),
+ (278, 'Sumatranlowlandrainforests', '278 (Sumatran lowland rain forests)'),
+ (279, 'Sumatranmontanerainforests', '279 (Sumatran montane rain forests)'),
+ (280, 'Sumatranpeatswampforests', '280 (Sumatran peat swamp forests)'),
+ (281, 'Sundalandheathforests', '281 (Sundaland heath forests)'),
+ (282, 'Sundarbansfreshwaterswampforests', '282 (Sundarbans freshwater swamp forests)'),
+ (283, 'Taiwansubtropicalevergreenforests', '283 (Taiwan subtropical evergreen forests)'),
+ (284, 'TenasserimSouthThailandsemievergreenrainforests', '284 (Tenasserim-South Thailand semi-evergreen rain forests)'),
+ (285, 'TonleSapfreshwaterswampforests', '285 (Tonle Sap freshwater swamp forests)'),
+ (286, 'TonleSapMekongpeatswampforests', '286 (Tonle Sap-Mekong peat swamp forests)'),
+ (287, 'UpperGangeticPlainsmoistdeciduousforests', '287 (Upper Gangetic Plains moist deciduous forests)'),
+ (288, 'WesternJavamontanerainforests', '288 (Western Java montane rain forests)'),
+ (289, 'WesternJavarainforests', '289 (Western Java rain forests)'),
+ (290, 'CentralDeccanPlateaudrydeciduousforests', '290 (Central Deccan Plateau dry deciduous forests)'),
+ (291, 'CentralIndochinadryforests', '291 (Central Indochina dry forests)'),
+ (292, 'ChhotaNagpurdrydeciduousforests', '292 (Chhota-Nagpur dry deciduous forests)'),
+ (293, 'EastDeccandryevergreenforests', '293 (East Deccan dry-evergreen forests)'),
+ (294, 'Irrawaddydryforests', '294 (Irrawaddy dry forests)'),
+ (295, 'KhathiarGirdrydeciduousforests', '295 (Khathiar-Gir dry deciduous forests)'),
+ (296, 'NarmadaValleydrydeciduousforests', '296 (Narmada Valley dry deciduous forests)'),
+ (297, 'NorthDeccandrydeciduousforests', '297 (North Deccan dry deciduous forests)'),
+ (298, 'SouthDeccanPlateaudrydeciduousforests', '298 (South Deccan Plateau dry deciduous forests)'),
+ (299, 'SoutheastIndochinadryevergreenforests', '299 (Southeast Indochina dry evergreen forests)'),
+ (300, 'SouthernVietnamlowlanddryforests', '300 (Southern Vietnam lowland dry forests)'),
+ (301, 'SriLankadryzonedryevergreenforests', '301 (Sri Lanka dry-zone dry evergreen forests)'),
+ (302, 'Himalayansubtropicalpineforests', '302 (Himalayan subtropical pine forests)'),
+ (303, 'Luzontropicalpineforests', '303 (Luzon tropical pine forests)'),
+ (304, 'NortheastIndiaMyanmarpineforests', '304 (Northeast India-Myanmar pine forests)'),
+ (305, 'Sumatrantropicalpineforests', '305 (Sumatran tropical pine forests)'),
+ (306, 'EasternHimalayanbroadleafforests', '306 (Eastern Himalayan broadleaf forests)'),
+ (307, 'NorthernTriangletemperateforests', '307 (Northern Triangle temperate forests)'),
+ (308, 'WesternHimalayanbroadleafforests', '308 (Western Himalayan broadleaf forests)'),
+ (309, 'EasternHimalayansubalpineconiferforests', '309 (Eastern Himalayan subalpine conifer forests)'),
+ (310, 'WesternHimalayansubalpineconiferforests', '310 (Western Himalayan subalpine conifer forests)'),
+ (311, 'TeraiDuarsavannaandgrasslands', '311 (Terai-Duar savanna and grasslands)'),
+ (312, 'RannofKutchseasonalsaltmarsh', '312 (Rann of Kutch seasonal salt marsh)'),
+ (313, 'Kinabalumontanealpinemeadows', '313 (Kinabalu montane alpine meadows)'),
+ (314, 'Aravalliwestthornscrubforests', '314 (Aravalli west thorn scrub forests)'),
+ (315, 'Deccanthornscrubforests', '315 (Deccan thorn scrub forests)'),
+ (316, 'GodavariKrishnamangroves', '316 (Godavari-Krishna mangroves)'),
+ (317, 'IndusValleydesert', '317 (Indus Valley desert)'),
+ (318, 'Thardesert', '318 (Thar desert)'),
+ (319, 'Indochinamangroves', '319 (Indochina mangroves)'),
+ (320, 'IndusRiverDeltaArabianSeamangroves', '320 (Indus River Delta-Arabian Sea mangroves)'),
+ (321, 'MyanmarCoastmangroves', '321 (Myanmar Coast mangroves)'),
+ (322, 'SundaShelfmangroves', '322 (Sunda Shelf mangroves)'),
+ (323, 'Sundarbansmangroves', '323 (Sundarbans mangroves)'),
+ (324, 'SonoranSinaloansubtropicaldryforest', '324 (Sonoran-Sinaloan subtropical dry forest)'),
+ (325, 'Bermudasubtropicalconiferforests', '325 (Bermuda subtropical conifer forests)'),
+ (326, 'SierraMadreOccidentalpineoakforests', '326 (Sierra Madre Occidental pine-oak forests)'),
+ (327, 'SierraMadreOrientalpineoakforests', '327 (Sierra Madre Oriental pine-oak forests)'),
+ (328, 'AlleghenyHighlandsforests', '328 (Allegheny Highlands forests)'),
+ (329, 'Appalachianmixedmesophyticforests', '329 (Appalachian mixed mesophytic forests)'),
+ (330, 'AppalachianPiedmontforests', '330 (Appalachian Piedmont forests)'),
+ (331, 'AppalachianBlueRidgeforests', '331 (Appalachian-Blue Ridge forests)'),
+ (332, 'EastCentralTexasforests', '332 (East Central Texas forests)'),
+ (333, 'EasternCanadianForestBorealtransition', '333 (Eastern Canadian Forest-Boreal transition)'),
+ (334, 'EasternGreatLakeslowlandforests', '334 (Eastern Great Lakes lowland forests)'),
+ (335, 'GulfofStLawrencelowlandforests', '335 (Gulf of St. Lawrence lowland forests)'),
+ (336, 'InteriorPlateauUSHardwoodForests', '336 (Interior Plateau US Hardwood Forests)'),
+ (337, 'Mississippilowlandforests', '337 (Mississippi lowland forests)'),
+ (338, 'NewEnglandAcadianforests', '338 (New England-Acadian forests)'),
+ (339, 'NortheastUSCoastalforests', '339 (Northeast US Coastal forests)'),
+ (340, 'OzarkHighlandsmixedforests', '340 (Ozark Highlands mixed forests)'),
+ (341, 'OzarkMountainforests', '341 (Ozark Mountain forests)'),
+ (342, 'SouthernGreatLakesforests', '342 (Southern Great Lakes forests)'),
+ (343, 'UpperMidwestUSforestsavannatransition', '343 (Upper Midwest US forest-savanna transition)'),
+ (344, 'WesternGreatLakesforests', '344 (Western Great Lakes forests)'),
+ (345, 'AlbertaBritishColumbiafoothillsforests', '345 (Alberta-British Columbia foothills forests)'),
+ (346, 'ArizonaMountainsforests', '346 (Arizona Mountains forests)'),
+ (347, 'Atlanticcoastalpinebarrens', '347 (Atlantic coastal pine barrens)'),
+ (348, 'BlueMountainsforests', '348 (Blue Mountains forests)'),
+ (349, 'BritishColumbiacoastalconiferforests', '349 (British Columbia coastal conifer forests)'),
+ (350, 'CentralBritishColumbiaMountainforests', '350 (Central British Columbia Mountain forests)'),
+ (351, 'CentralPacificNorthwestcoastalforests', '351 (Central Pacific Northwest coastal forests)'),
+ (352, 'CentralSouthernCascadesForests', '352 (Central-Southern Cascades Forests)'),
+ (353, 'ColoradoRockiesforests', '353 (Colorado Rockies forests)'),
+ (354, 'EasternCascadesforests', '354 (Eastern Cascades forests)'),
+ (355, 'FraserPlateauandBasinconiferforests', '355 (Fraser Plateau and Basin conifer forests)'),
+ (356, 'GreatBasinmontaneforests', '356 (Great Basin montane forests)'),
+ (357, 'KlamathSiskiyouforests', '357 (Klamath-Siskiyou forests)'),
+ (358, 'NorthCascadesconiferforests', '358 (North Cascades conifer forests)'),
+ (359, 'NorthernCaliforniacoastalforests', '359 (Northern California coastal forests)'),
+ (360, 'NorthernPacificAlaskancoastalforests', '360 (Northern Pacific Alaskan coastal forests)'),
+ (361, 'NorthernRockiesconiferforests', '361 (Northern Rockies conifer forests)'),
+ (362, 'Okanogandryforests', '362 (Okanogan dry forests)'),
+ (363, 'PineyWoods', '363 (Piney Woods)'),
+ (364, 'Pugetlowlandforests', '364 (Puget lowland forests)'),
+ (365, 'QueenCharlotteIslandsconiferforests', '365 (Queen Charlotte Islands conifer forests)'),
+ (366, 'SierraNevadaforests', '366 (Sierra Nevada forests)'),
+ (367, 'SouthCentralRockiesforests', '367 (South Central Rockies forests)'),
+ (368, 'WasatchandUintamontaneforests', '368 (Wasatch and Uinta montane forests)'),
+ (369, 'AlaskaPeninsulamontanetaiga', '369 (Alaska Peninsula montane taiga)'),
+ (370, 'CentralCanadianShieldforests', '370 (Central Canadian Shield forests)'),
+ (371, 'CookInlettaiga', '371 (Cook Inlet taiga)'),
+ (372, 'CopperPlateautaiga', '372 (Copper Plateau taiga)'),
+ (373, 'EasternCanadianforests', '373 (Eastern Canadian forests)'),
+ (374, 'EasternCanadianShieldtaiga', '374 (Eastern Canadian Shield taiga)'),
+ (375, 'InteriorAlaskaYukonlowlandtaiga', '375 (Interior Alaska-Yukon lowland taiga)'),
+ (376, 'MidCanadaBorealPlainsforests', '376 (Mid-Canada Boreal Plains forests)'),
+ (377, 'MidwestCanadianShieldforests', '377 (Midwest Canadian Shield forests)'),
+ (378, 'MuskwaSlaveLaketaiga', '378 (Muskwa-Slave Lake taiga)'),
+ (379, 'NorthernCanadianShieldtaiga', '379 (Northern Canadian Shield taiga)'),
+ (380, 'NorthernCordilleraforests', '380 (Northern Cordillera forests)'),
+ (381, 'NorthwestTerritoriestaiga', '381 (Northwest Territories taiga)'),
+ (382, 'SouthernHudsonBaytaiga', '382 (Southern Hudson Bay taiga)'),
+ (383, 'WatsonHighlandstaiga', '383 (Watson Highlands taiga)'),
+ (384, 'WesternGulfcoastalgrasslands', '384 (Western Gulf coastal grasslands)'),
+ (385, 'CaliforniaCentralValleygrasslands', '385 (California Central Valley grasslands)'),
+ (386, 'CanadianAspenforestsandparklands', '386 (Canadian Aspen forests and parklands)'),
+ (387, 'CentralUSforestgrasslandstransition', '387 (Central US forest-grasslands transition)'),
+ (388, 'CentralTallgrassprairie', '388 (Central Tallgrass prairie)'),
+ (389, 'CentralSouthernUSmixedgrasslands', '389 (Central-Southern US mixed grasslands)'),
+ (390, 'CrossTimberssavannawoodland', '390 (Cross-Timbers savanna-woodland)'),
+ (391, 'EdwardsPlateausavanna', '391 (Edwards Plateau savanna)'),
+ (392, 'FlintHillstallgrassprairie', '392 (Flint Hills tallgrass prairie)'),
+ (393, 'MidAtlanticUScoastalsavannas', '393 (Mid-Atlantic US coastal savannas)'),
+ (394, 'MontanaValleyandFoothillgrasslands', '394 (Montana Valley and Foothill grasslands)'),
+ (395, 'NebraskaSandHillsmixedgrasslands', '395 (Nebraska Sand Hills mixed grasslands)'),
+ (396, 'NorthernShortgrassprairie', '396 (Northern Shortgrass prairie)'),
+ (397, 'NorthernTallgrassprairie', '397 (Northern Tallgrass prairie)'),
+ (398, 'Palouseprairie', '398 (Palouse prairie)'),
+ (399, 'SoutheastUSconifersavannas', '399 (Southeast US conifer savannas)'),
+ (400, 'SoutheastUSmixedwoodlandsandsavannas', '400 (Southeast US mixed woodlands and savannas)'),
+ (401, 'Texasblacklandprairies', '401 (Texas blackland prairies)'),
+ (402, 'Westernshortgrassprairie', '402 (Western shortgrass prairie)'),
+ (403, 'WillametteValleyoaksavanna', '403 (Willamette Valley oak savanna)'),
+ (404, 'AhklunandKilbuckUplandTundra', '404 (Ahklun and Kilbuck Upland Tundra)'),
+ (405, 'AlaskaStEliasRangetundra', '405 (Alaska-St. Elias Range tundra)'),
+ (406, 'AleutianIslandstundra', '406 (Aleutian Islands tundra)'),
+ (407, 'Arcticcoastaltundra', '407 (Arctic coastal tundra)'),
+ (408, 'Arcticfoothillstundra', '408 (Arctic foothills tundra)'),
+ (409, 'Beringialowlandtundra', '409 (Beringia lowland tundra)'),
+ (410, 'Beringiauplandtundra', '410 (Beringia upland tundra)'),
+ (411, 'BrooksBritishRangetundra', '411 (Brooks-British Range tundra)'),
+ (412, 'CanadianHighArctictundra', '412 (Canadian High Arctic tundra)'),
+ (413, 'CanadianLowArctictundra', '413 (Canadian Low Arctic tundra)'),
+ (414, 'CanadianMiddleArcticTundra', '414 (Canadian Middle Arctic Tundra)'),
+ (415, 'DavisHighlandstundra', '415 (Davis Highlands tundra)'),
+ (416, 'InteriorYukonAlaskaalpinetundra', '416 (Interior Yukon-Alaska alpine tundra)'),
+ (417, 'KalaallitNunaatArcticsteppe', '417 (Kalaallit Nunaat Arctic steppe)'),
+ (418, 'KalaallitNunaatHighArctictundra', '418 (Kalaallit Nunaat High Arctic tundra)'),
+ (419, 'OgilvieMacKenziealpinetundra', '419 (Ogilvie-MacKenzie alpine tundra)'),
+ (420, 'PacificCoastalMountainicefieldsandtundra', '420 (Pacific Coastal Mountain icefields and tundra)'),
+ (421, 'TorngatMountaintundra', '421 (Torngat Mountain tundra)'),
+ (422, 'Californiacoastalsageandchaparral', '422 (California coastal sage and chaparral)'),
+ (423, 'Californiainteriorchaparralandwoodlands', '423 (California interior chaparral and woodlands)'),
+ (424, 'Californiamontanechaparralandwoodlands', '424 (California montane chaparral and woodlands)'),
+ (425, 'SantaLuciaMontaneChaparralAndWoodlands', '425 (Santa Lucia Montane Chaparral & Woodlands)'),
+ (426, 'BajaCaliforniadesert', '426 (Baja California desert)'),
+ (427, 'CentralMexicanmatorral', '427 (Central Mexican matorral)'),
+ (428, 'Chihuahuandesert', '428 (Chihuahuan desert)'),
+ (429, 'ColoradoPlateaushrublands', '429 (Colorado Plateau shrublands)'),
+ (430, 'GreatBasinshrubsteppe', '430 (Great Basin shrub steppe)'),
+ (431, 'GulfofCaliforniaxericscrub', '431 (Gulf of California xeric scrub)'),
+ (432, 'MesetaCentralmatorral', '432 (Meseta Central matorral)'),
+ (433, 'Mojavedesert', '433 (Mojave desert)'),
+ (434, 'SnakeColumbiashrubsteppe', '434 (Snake-Columbia shrub steppe)'),
+ (435, 'Sonorandesert', '435 (Sonoran desert)'),
+ (436, 'Tamaulipanmatorral', '436 (Tamaulipan matorral)'),
+ (437, 'Tamaulipanmezquital', '437 (Tamaulipan mezquital)'),
+ (438, 'WyomingBasinshrubsteppe', '438 (Wyoming Basin shrub steppe)'),
+ (439, 'AltoParanaAtlanticforests', '439 (Alto Paraná Atlantic forests)'),
+ (440, 'Araucariamoistforests', '440 (Araucaria moist forests)'),
+ (441, 'AtlanticCoastrestingas', '441 (Atlantic Coast restingas)'),
+ (442, 'Bahiacoastalforests', '442 (Bahia coastal forests)'),
+ (443, 'Bahiainteriorforests', '443 (Bahia interior forests)'),
+ (444, 'BolivianYungas', '444 (Bolivian Yungas)'),
+ (445, 'CaatingaEnclavesmoistforests', '445 (Caatinga Enclaves moist forests)'),
+ (446, 'Caquetamoistforests', '446 (Caqueta moist forests)'),
+ (447, 'Catatumbomoistforests', '447 (Catatumbo moist forests)'),
+ (448, 'CaucaValleymontaneforests', '448 (Cauca Valley montane forests)'),
+ (449, 'CayosMiskitosSanAndresandProvidenciamoistforests', '449 (Cayos Miskitos-San Andrés and Providencia moist forests)'),
+ (450, 'CentralAmericanAtlanticmoistforests', '450 (Central American Atlantic moist forests)'),
+ (451, 'CentralAmericanmontaneforests', '451 (Central American montane forests)'),
+ (452, 'Chiapasmontaneforests', '452 (Chiapas montane forests)'),
+ (453, 'Chimalapasmontaneforests', '453 (Chimalapas montane forests)'),
+ (454, 'ChocoDarienmoistforests', '454 (Chocó-Darién moist forests)'),
+ (455, 'CocosIslandmoistforests', '455 (Cocos Island moist forests)'),
+ (456, 'CordilleraLaCostamontaneforests', '456 (Cordillera La Costa montane forests)'),
+ (457, 'CordilleraOrientalmontaneforests', '457 (Cordillera Oriental montane forests)'),
+ (458, 'CostaRicanseasonalmoistforests', '458 (Costa Rican seasonal moist forests)'),
+ (459, 'Cubanmoistforests', '459 (Cuban moist forests)'),
+ (460, 'EasternCordilleraRealmontaneforests', '460 (Eastern Cordillera Real montane forests)'),
+ (461, 'EasternPanamanianmontaneforests', '461 (Eastern Panamanian montane forests)'),
+ (462, 'FernandodeNoronhaAtoldasRocasmoistforests', '462 (Fernando de Noronha-Atol das Rocas moist forests)'),
+ (463, 'Guiananfreshwaterswampforests', '463 (Guianan freshwater swamp forests)'),
+ (464, 'GuiananHighlandsmoistforests', '464 (Guianan Highlands moist forests)'),
+ (465, 'Guiananlowlandmoistforests', '465 (Guianan lowland moist forests)'),
+ (466, 'Guiananpiedmontmoistforests', '466 (Guianan piedmont moist forests)'),
+ (467, 'Gurupavarzea', '467 (Gurupa várzea)'),
+ (468, 'Hispaniolanmoistforests', '468 (Hispaniolan moist forests)'),
+ (469, 'Iquitosvarzea', '469 (Iquitos várzea)'),
+ (470, 'IsthmianAtlanticmoistforests', '470 (Isthmian-Atlantic moist forests)'),
+ (471, 'IsthmianPacificmoistforests', '471 (Isthmian-Pacific moist forests)'),
+ (472, 'Jamaicanmoistforests', '472 (Jamaican moist forests)'),
+ (473, 'JapuraSolimoesNegromoistforests', '473 (Japurá-Solimões-Negro moist forests)'),
+ (474, 'JuruaPurusmoistforests', '474 (Juruá-Purus moist forests)'),
+ (475, 'LeewardIslandsmoistforests', '475 (Leeward Islands moist forests)'),
+ (476, 'MadeiraTapajosmoistforests', '476 (Madeira-Tapajós moist forests)'),
+ (477, 'MagdalenaValleymontaneforests', '477 (Magdalena Valley montane forests)'),
+ (478, 'MagdalenaUrabamoistforests', '478 (Magdalena-Urabá moist forests)'),
+ (479, 'Maranondryforests', '479 (Marañón dry forests)'),
+ (480, 'Marajovarzea', '480 (Marajó várzea)'),
+ (481, 'MatoGrossotropicaldryforests', '481 (Mato Grosso tropical dry forests)'),
+ (482, 'MonteAlegrevarzea', '482 (Monte Alegre várzea)'),
+ (483, 'Napomoistforests', '483 (Napo moist forests)'),
+ (484, 'NegroBrancomoistforests', '484 (Negro-Branco moist forests)'),
+ (485, 'NortheastBrazilrestingas', '485 (Northeast Brazil restingas)'),
+ (486, 'NorthwestAndeanmontaneforests', '486 (Northwest Andean montane forests)'),
+ (487, 'Oaxacanmontaneforests', '487 (Oaxacan montane forests)'),
+ (488, 'OrinocoDeltaswampforests', '488 (Orinoco Delta swamp forests)'),
+ (489, 'PantanosdeCentla', '489 (Pantanos de Centla)'),
+ (490, 'PantepuiforestsAndshrublands', '490 (Pantepui forests & shrublands)'),
+ (491, 'Pernambucocoastalforests', '491 (Pernambuco coastal forests)'),
+ (492, 'Pernambucointeriorforests', '492 (Pernambuco interior forests)'),
+ (493, 'PeruvianYungas', '493 (Peruvian Yungas)'),
+ (494, 'PetenVeracruzmoistforests', '494 (Petén-Veracruz moist forests)'),
+ (495, 'PuertoRicanmoistforests', '495 (Puerto Rican moist forests)'),
+ (496, 'Purusvarzea', '496 (Purus várzea)'),
+ (497, 'PurusMadeiramoistforests', '497 (Purus-Madeira moist forests)'),
+ (498, 'RioNegrocampinarana', '498 (Rio Negro campinarana)'),
+ (499, 'SantaMartamontaneforests', '499 (Santa Marta montane forests)'),
+ (500, 'SerradoMarcoastalforests', '500 (Serra do Mar coastal forests)'),
+ (501, 'SierradelosTuxtlas', '501 (Sierra de los Tuxtlas)'),
+ (502, 'SierraMadredeChiapasmoistforests', '502 (Sierra Madre de Chiapas moist forests)'),
+ (503, 'SolimoesJapuramoistforests', '503 (Solimões-Japurá moist forests)'),
+ (504, 'SouthernAndeanYungas', '504 (Southern Andean Yungas)'),
+ (505, 'SouthwestAmazonmoistforests', '505 (Southwest Amazon moist forests)'),
+ (506, 'Talamancanmontaneforests', '506 (Talamancan montane forests)'),
+ (507, 'TapajosXingumoistforests', '507 (Tapajós-Xingu moist forests)'),
+ (508, 'TocantinsPindaremoistforests', '508 (Tocantins/Pindare moist forests)'),
+ (509, 'TrindadeMartinVazIslandstropicalforests', '509 (Trindade-Martin Vaz Islands tropical forests)'),
+ (510, 'TrinidadandTobagomoistforest', '510 (Trinidad and Tobago moist forest)'),
+ (511, 'UatumaTrombetasmoistforests', '511 (Uatumã-Trombetas moist forests)'),
+ (512, 'Ucayalimoistforests', '512 (Ucayali moist forests)'),
+ (513, 'VenezuelanAndesmontaneforests', '513 (Venezuelan Andes montane forests)'),
+ (514, 'Veracruzmoistforests', '514 (Veracruz moist forests)'),
+ (515, 'Veracruzmontaneforests', '515 (Veracruz montane forests)'),
+ (516, 'WesternEcuadormoistforests', '516 (Western Ecuador moist forests)'),
+ (517, 'WindwardIslandsmoistforests', '517 (Windward Islands moist forests)'),
+ (518, 'XinguTocantinsAraguaiamoistforests', '518 (Xingu-Tocantins-Araguaia moist forests)'),
+ (519, 'Yucatanmoistforests', '519 (Yucatán moist forests)'),
+ (520, 'ApureVillavicenciodryforests', '520 (Apure-Villavicencio dry forests)'),
+ (521, 'Bajiodryforests', '521 (Bajío dry forests)'),
+ (522, 'Balsasdryforests', '522 (Balsas dry forests)'),
+ (523, 'Bolivianmontanedryforests', '523 (Bolivian montane dry forests)'),
+ (524, 'BrazilianAtlanticdryforests', '524 (Brazilian Atlantic dry forests)'),
+ (525, 'Caatinga', '525 (Caatinga)'),
+ (526, 'CaucaValleydryforests', '526 (Cauca Valley dry forests)'),
+ (527, 'CentralAmericandryforests', '527 (Central American dry forests)'),
+ (528, 'ChiapasDepressiondryforests', '528 (Chiapas Depression dry forests)'),
+ (529, 'Chiquitanodryforests', '529 (Chiquitano dry forests)'),
+ (530, 'Cubandryforests', '530 (Cuban dry forests)'),
+ (531, 'Ecuadoriandryforests', '531 (Ecuadorian dry forests)'),
+ (532, 'Hispaniolandryforests', '532 (Hispaniolan dry forests)'),
+ (533, 'IslasRevillagigedodryforests', '533 (Islas Revillagigedo dry forests)'),
+ (534, 'Jaliscodryforests', '534 (Jalisco dry forests)'),
+ (535, 'Jamaicandryforests', '535 (Jamaican dry forests)'),
+ (536, 'LaraFalcondryforests', '536 (Lara-Falcón dry forests)'),
+ (537, 'LesserAntilleandryforests', '537 (Lesser Antillean dry forests)'),
+ (538, 'MagdalenaValleydryforests', '538 (Magdalena Valley dry forests)'),
+ (539, 'Maracaibodryforests', '539 (Maracaibo dry forests)'),
+ (540, 'MaranhaoBabacuforests', '540 (Maranhão Babaçu forests)'),
+ (541, 'Panamaniandryforests', '541 (Panamanian dry forests)'),
+ (542, 'Patiavalleydryforests', '542 (Patía valley dry forests)'),
+ (543, 'PuertoRicandryforests', '543 (Puerto Rican dry forests)'),
+ (544, 'SierradelaLagunadryforests', '544 (Sierra de la Laguna dry forests)'),
+ (545, 'Sinaloandryforests', '545 (Sinaloan dry forests)'),
+ (546, 'SinuValleydryforests', '546 (Sinú Valley dry forests)'),
+ (547, 'SouthernPacificdryforests', '547 (Southern Pacific dry forests)'),
+ (548, 'TrinidadandTobagodryforest', '548 (Trinidad and Tobago dry forest)'),
+ (549, 'TumbesPiuradryforests', '549 (Tumbes-Piura dry forests)'),
+ (550, 'Veracruzdryforests', '550 (Veracruz dry forests)'),
+ (551, 'Yucatandryforests', '551 (Yucatán dry forests)'),
+ (552, 'Bahamianpineyards', '552 (Bahamian pineyards)'),
+ (553, 'CentralAmericanpineoakforests', '553 (Central American pine-oak forests)'),
+ (554, 'Cubanpineforests', '554 (Cuban pine forests)'),
+ (555, 'Hispaniolanpineforests', '555 (Hispaniolan pine forests)'),
+ (556, 'SierradelaLagunapineoakforests', '556 (Sierra de la Laguna pine-oak forests)'),
+ (557, 'SierraMadredeOaxacapineoakforests', '557 (Sierra Madre de Oaxaca pine-oak forests)'),
+ (558, 'SierraMadredelSurpineoakforests', '558 (Sierra Madre del Sur pine-oak forests)'),
+ (559, 'TransMexicanVolcanicBeltpineoakforests', '559 (Trans-Mexican Volcanic Belt pine-oak forests)'),
+ (560, 'JuanFernandezIslandstemperateforests', '560 (Juan Fernández Islands temperate forests)'),
+ (561, 'Magellanicsubpolarforests', '561 (Magellanic subpolar forests)'),
+ (562, 'SanFelixSanAmbrosioIslandstemperateforests', '562 (San Félix-San Ambrosio Islands temperate forests)'),
+ (563, 'Valdiviantemperateforests', '563 (Valdivian temperate forests)'),
+ (564, 'Belizianpinesavannas', '564 (Belizian pine savannas)'),
+ (565, 'Benisavanna', '565 (Beni savanna)'),
+ (566, 'CamposRupestresmontanesavanna', '566 (Campos Rupestres montane savanna)'),
+ (567, 'Cerrado', '567 (Cerrado)'),
+ (568, 'ClippertonIslandshrubandgrasslands', '568 (Clipperton Island shrub and grasslands)'),
+ (569, 'DryChaco', '569 (Dry Chaco)'),
+ (570, 'Guianansavanna', '570 (Guianan savanna)'),
+ (571, 'HumidChaco', '571 (Humid Chaco)'),
+ (572, 'Llanos', '572 (Llanos)'),
+ (573, 'Miskitopineforests', '573 (Miskito pine forests)'),
+ (574, 'Uruguayansavanna', '574 (Uruguayan savanna)'),
+ (575, 'Espinal', '575 (Espinal)'),
+ (576, 'HumidPampas', '576 (Humid Pampas)'),
+ (577, 'LowMonte', '577 (Low Monte)'),
+ (578, 'Patagoniansteppe', '578 (Patagonian steppe)'),
+ (579, 'Cubanwetlands', '579 (Cuban wetlands)'),
+ (580, 'Enriquillowetlands', '580 (Enriquillo wetlands)'),
+ (581, 'Evergladesfloodedgrasslands', '581 (Everglades flooded grasslands)'),
+ (582, 'Guayaquilfloodedgrasslands', '582 (Guayaquil flooded grasslands)'),
+ (583, 'Orinocowetlands', '583 (Orinoco wetlands)'),
+ (584, 'Pantanal', '584 (Pantanal)'),
+ (585, 'Paranafloodedsavanna', '585 (Paraná flooded savanna)'),
+ (586, 'SouthernConeMesopotamiansavanna', '586 (Southern Cone Mesopotamian savanna)'),
+ (587, 'CentralAndeandrypuna', '587 (Central Andean dry puna)'),
+ (588, 'CentralAndeanpuna', '588 (Central Andean puna)'),
+ (589, 'CentralAndeanwetpuna', '589 (Central Andean wet puna)'),
+ (590, 'CordilleraCentralparamo', '590 (Cordillera Central páramo)'),
+ (591, 'CordilleradeMeridaparamo', '591 (Cordillera de Merida páramo)'),
+ (592, 'HighMonte', '592 (High Monte)'),
+ (593, 'NorthernAndeanparamo', '593 (Northern Andean páramo)'),
+ (594, 'SantaMartaparamo', '594 (Santa Marta páramo)'),
+ (595, 'SouthernAndeansteppe', '595 (Southern Andean steppe)'),
+ (596, 'ChileanMatorral', '596 (Chilean Matorral)'),
+ (597, 'ArayaandPariaxericscrub', '597 (Araya and Paria xeric scrub)'),
+ (598, 'Atacamadesert', '598 (Atacama desert)'),
+ (599, 'Caribbeanshrublands', '599 (Caribbean shrublands)'),
+ (600, 'Cubancactusscrub', '600 (Cuban cactus scrub)'),
+ (601, 'GalapagosIslandsxericscrub', '601 (Galápagos Islands xeric scrub)'),
+ (602, 'GuajiraBarranquillaxericscrub', '602 (Guajira-Barranquilla xeric scrub)'),
+ (603, 'LaCostaxericshrublands', '603 (La Costa xeric shrublands)'),
+ (604, 'MalpeloIslandxericscrub', '604 (Malpelo Island xeric scrub)'),
+ (605, 'MotaguaValleythornscrub', '605 (Motagua Valley thornscrub)'),
+ (606, 'Paraguanaxericscrub', '606 (Paraguaná xeric scrub)'),
+ (607, 'SanLucanxericscrub', '607 (San Lucan xeric scrub)'),
+ (608, 'Sechuradesert', '608 (Sechura desert)'),
+ (609, 'StPeterandStPaulRocks', '609 (St. Peter and St. Paul Rocks)'),
+ (610, 'TehuacanValleymatorral', '610 (Tehuacán Valley matorral)'),
+ (611, 'AmazonOrinocoSouthernCaribbeanmangroves', '611 (Amazon-Orinoco-Southern Caribbean mangroves)'),
+ (612, 'BahamianAntilleanmangroves', '612 (Bahamian-Antillean mangroves)'),
+ (613, 'MesoamericanGulfCaribbeanmangroves', '613 (Mesoamerican Gulf-Caribbean mangroves)'),
+ (614, 'NorthernMesoamericanPacificmangroves', '614 (Northern Mesoamerican Pacific mangroves)'),
+ (615, 'SouthAmericanPacificmangroves', '615 (South American Pacific mangroves)'),
+ (616, 'SouthernAtlanticBrazilianmangroves', '616 (Southern Atlantic Brazilian mangroves)'),
+ (617, 'SouthernMesoamericanPacificmangroves', '617 (Southern Mesoamerican Pacific mangroves)'),
+ (618, 'Carolinestropicalmoistforests', '618 (Carolines tropical moist forests)'),
+ (619, 'CentralPolynesiantropicalmoistforests', '619 (Central Polynesian tropical moist forests)'),
+ (620, 'CookIslandstropicalmoistforests', '620 (Cook Islands tropical moist forests)'),
+ (621, 'EasternMicronesiatropicalmoistforests', '621 (Eastern Micronesia tropical moist forests)'),
+ (622, 'Fijitropicalmoistforests', '622 (Fiji tropical moist forests)'),
+ (623, 'Hawaiitropicalmoistforests', '623 (Hawai''i tropical moist forests)'),
+ (624, 'KermadecIslandssubtropicalmoistforests', '624 (Kermadec Islands subtropical moist forests)'),
+ (625, 'Marquesastropicalmoistforests', '625 (Marquesas tropical moist forests)'),
+ (626, 'Ogasawarasubtropicalmoistforests', '626 (Ogasawara subtropical moist forests)'),
+ (627, 'Palautropicalmoistforests', '627 (Palau tropical moist forests)'),
+ (628, 'RapaNuiandSalayGomezsubtropicalforests', '628 (Rapa Nui and Sala y Gómez subtropical forests)'),
+ (629, 'Samoantropicalmoistforests', '629 (Samoan tropical moist forests)'),
+ (630, 'SocietyIslandstropicalmoistforests', '630 (Society Islands tropical moist forests)'),
+ (631, 'Tongantropicalmoistforests', '631 (Tongan tropical moist forests)'),
+ (632, 'Tuamotutropicalmoistforests', '632 (Tuamotu tropical moist forests)'),
+ (633, 'Tubuaitropicalmoistforests', '633 (Tubuai tropical moist forests)'),
+ (634, 'WesternPolynesiantropicalmoistforests', '634 (Western Polynesian tropical moist forests)'),
+ (635, 'Fijitropicaldryforests', '635 (Fiji tropical dry forests)'),
+ (636, 'Hawaiitropicaldryforests', '636 (Hawai''i tropical dry forests)'),
+ (637, 'Marianastropicaldryforests', '637 (Marianas tropical dry forests)'),
+ (638, 'Yaptropicaldryforests', '638 (Yap tropical dry forests)'),
+ (639, 'Hawaiitropicalhighshrublands', '639 (Hawai''i tropical high shrublands)'),
+ (640, 'Hawaiitropicallowshrublands', '640 (Hawai''i tropical low shrublands)'),
+ (641, 'NorthwestHawaiiscrub', '641 (Northwest Hawai''i scrub)'),
+ (642, 'GuizhouPlateaubroadleafandmixedforests', '642 (Guizhou Plateau broadleaf and mixed forests)'),
+ (643, 'YunnanPlateausubtropicalevergreenforests', '643 (Yunnan Plateau subtropical evergreen forests)'),
+ (644, 'Appeninedeciduousmontaneforests', '644 (Appenine deciduous montane forests)'),
+ (645, 'Azorestemperatemixedforests', '645 (Azores temperate mixed forests)'),
+ (646, 'Balkanmixedforests', '646 (Balkan mixed forests)'),
+ (647, 'Balticmixedforests', '647 (Baltic mixed forests)'),
+ (648, 'Cantabrianmixedforests', '648 (Cantabrian mixed forests)'),
+ (649, 'CaspianHyrcanianmixedforests', '649 (Caspian Hyrcanian mixed forests)'),
+ (650, 'Caucasusmixedforests', '650 (Caucasus mixed forests)'),
+ (651, 'Celticbroadleafforests', '651 (Celtic broadleaf forests)'),
+ (652, 'CentralAnatoliansteppeandwoodlands', '652 (Central Anatolian steppe and woodlands)'),
+ (653, 'CentralChinaLoessPlateaumixedforests', '653 (Central China Loess Plateau mixed forests)'),
+ (654, 'CentralEuropeanmixedforests', '654 (Central European mixed forests)'),
+ (655, 'CentralKoreandeciduousforests', '655 (Central Korean deciduous forests)'),
+ (656, 'ChangbaiMountainsmixedforests', '656 (Changbai Mountains mixed forests)'),
+ (657, 'ChangjiangPlainevergreenforests', '657 (Changjiang Plain evergreen forests)'),
+ (658, 'CrimeanSubmediterraneanforestcomplex', '658 (Crimean Submediterranean forest complex)'),
+ (659, 'DabaMountainsevergreenforests', '659 (Daba Mountains evergreen forests)'),
+ (660, 'DinaricMountainsmixedforests', '660 (Dinaric Mountains mixed forests)'),
+ (661, 'EastEuropeanforeststeppe', '661 (East European forest steppe)'),
+ (662, 'EasternAnatoliandeciduousforests', '662 (Eastern Anatolian deciduous forests)'),
+ (663, 'EnglishLowlandsbeechforests', '663 (English Lowlands beech forests)'),
+ (664, 'EuropeanAtlanticmixedforests', '664 (European Atlantic mixed forests)'),
+ (665, 'EuxineColchicbroadleafforests', '665 (Euxine-Colchic broadleaf forests)'),
+ (666, 'Hokkaidodeciduousforests', '666 (Hokkaido deciduous forests)'),
+ (667, 'HuangHePlainmixedforests', '667 (Huang He Plain mixed forests)'),
+ (668, 'Madeiraevergreenforests', '668 (Madeira evergreen forests)'),
+ (669, 'Manchurianmixedforests', '669 (Manchurian mixed forests)'),
+ (670, 'Nihonkaievergreenforests', '670 (Nihonkai evergreen forests)'),
+ (671, 'Nihonkaimontanedeciduousforests', '671 (Nihonkai montane deciduous forests)'),
+ (672, 'NorthAtlanticmoistmixedforests', '672 (North Atlantic moist mixed forests)'),
+ (673, 'NortheastChinaPlaindeciduousforests', '673 (Northeast China Plain deciduous forests)'),
+ (674, 'Pannonianmixedforests', '674 (Pannonian mixed forests)'),
+ (675, 'PoBasinmixedforests', '675 (Po Basin mixed forests)'),
+ (676, 'Pyreneesconiferandmixedforests', '676 (Pyrenees conifer and mixed forests)'),
+ (677, 'QinLingMountainsdeciduousforests', '677 (Qin Ling Mountains deciduous forests)'),
+ (678, 'Rodopemontanemixedforests', '678 (Rodope montane mixed forests)'),
+ (679, 'Sarmaticmixedforests', '679 (Sarmatic mixed forests)'),
+ (680, 'SichuanBasinevergreenbroadleafforests', '680 (Sichuan Basin evergreen broadleaf forests)'),
+ (681, 'SouthernKoreaevergreenforests', '681 (Southern Korea evergreen forests)'),
+ (682, 'Taiheiyoevergreenforests', '682 (Taiheiyo evergreen forests)'),
+ (683, 'Taiheiyomontanedeciduousforests', '683 (Taiheiyo montane deciduous forests)'),
+ (684, 'TarimBasindeciduousforestsandsteppe', '684 (Tarim Basin deciduous forests and steppe)'),
+ (685, 'Ussuribroadleafandmixedforests', '685 (Ussuri broadleaf and mixed forests)'),
+ (686, 'WesternEuropeanbroadleafforests', '686 (Western European broadleaf forests)'),
+ (687, 'WesternSiberianhemiborealforests', '687 (Western Siberian hemiboreal forests)'),
+ (688, 'ZagrosMountainsforeststeppe', '688 (Zagros Mountains forest steppe)'),
+ (689, 'Alpsconiferandmixedforests', '689 (Alps conifer and mixed forests)'),
+ (690, 'Altaimontaneforestandforeststeppe', '690 (Altai montane forest and forest steppe)'),
+ (691, 'Caledonconiferforests', '691 (Caledon conifer forests)'),
+ (692, 'Carpathianmontaneforests', '692 (Carpathian montane forests)'),
+ (693, 'DaHingganDzhagdyMountainsconiferforests', '693 (Da Hinggan-Dzhagdy Mountains conifer forests)'),
+ (694, 'EastAfghanmontaneconiferforests', '694 (East Afghan montane conifer forests)'),
+ (695, 'ElburzRangeforeststeppe', '695 (Elburz Range forest steppe)'),
+ (696, 'Helanshanmontaneconiferforests', '696 (Helanshan montane conifer forests)'),
+ (697, 'HengduanMountainssubalpineconiferforests', '697 (Hengduan Mountains subalpine conifer forests)'),
+ (698, 'Hokkaidomontaneconiferforests', '698 (Hokkaido montane conifer forests)'),
+ (699, 'Honshualpineconiferforests', '699 (Honshu alpine conifer forests)'),
+ (700, 'KhangaiMountainsconiferforests', '700 (Khangai Mountains conifer forests)'),
+ (701, 'Mediterraneanconiferandmixedforests', '701 (Mediterranean conifer and mixed forests)'),
+ (702, 'NortheastHimalayansubalpineconiferforests', '702 (Northeast Himalayan subalpine conifer forests)'),
+ (703, 'NorthernAnatolianconiferanddeciduousforests', '703 (Northern Anatolian conifer and deciduous forests)'),
+ (704, 'NujiangLangcangGorgealpineconiferandmixedforests', '704 (Nujiang Langcang Gorge alpine conifer and mixed forests)'),
+ (705, 'QilianMountainsconiferforests', '705 (Qilian Mountains conifer forests)'),
+ (706, 'QionglaiMinshanconiferforests', '706 (Qionglai-Minshan conifer forests)'),
+ (707, 'Sayanmontaneconiferforests', '707 (Sayan montane conifer forests)'),
+ (708, 'Scandinaviancoastalconiferforests', '708 (Scandinavian coastal conifer forests)'),
+ (709, 'TianShanmontaneconiferforests', '709 (Tian Shan montane conifer forests)'),
+ (710, 'EastSiberiantaiga', '710 (East Siberian taiga)'),
+ (711, 'Icelandborealbirchforestsandalpinetundra', '711 (Iceland boreal birch forests and alpine tundra)'),
+ (712, 'Kamchatkataiga', '712 (Kamchatka taiga)'),
+ (713, 'KamchatkaKurilemeadowsandsparseforests', '713 (Kamchatka-Kurile meadows and sparse forests)'),
+ (714, 'NortheastSiberiantaiga', '714 (Northeast Siberian taiga)'),
+ (715, 'OkhotskManchuriantaiga', '715 (Okhotsk-Manchurian taiga)'),
+ (716, 'SakhalinIslandtaiga', '716 (Sakhalin Island taiga)'),
+ (717, 'ScandinavianandRussiantaiga', '717 (Scandinavian and Russian taiga)'),
+ (718, 'TransBaikalconiferforests', '718 (Trans-Baikal conifer forests)'),
+ (719, 'Uralsmontaneforestandtaiga', '719 (Urals montane forest and taiga)'),
+ (720, 'WestSiberiantaiga', '720 (West Siberian taiga)'),
+ (721, 'AlaiWesternTianShansteppe', '721 (Alai-Western Tian Shan steppe)'),
+ (722, 'AlHajarfoothillxericwoodlandsandshrublands', '722 (Al-Hajar foothill xeric woodlands and shrublands)'),
+ (723, 'AlHajarmontanewoodlandsandshrublands', '723 (Al-Hajar montane woodlands and shrublands)'),
+ (724, 'Altaisteppeandsemidesert', '724 (Altai steppe and semi-desert)'),
+ (725, 'CentralAnatoliansteppe', '725 (Central Anatolian steppe)'),
+ (726, 'Daurianforeststeppe', '726 (Daurian forest steppe)'),
+ (727, 'EasternAnatolianmontanesteppe', '727 (Eastern Anatolian montane steppe)'),
+ (728, 'EminValleysteppe', '728 (Emin Valley steppe)'),
+ (729, 'FaroeIslandsborealgrasslands', '729 (Faroe Islands boreal grasslands)'),
+ (730, 'GissaroAlaiopenwoodlands', '730 (Gissaro-Alai open woodlands)'),
+ (731, 'Kazakhforeststeppe', '731 (Kazakh forest steppe)'),
+ (732, 'Kazakhsteppe', '732 (Kazakh steppe)'),
+ (733, 'Kazakhuplandsteppe', '733 (Kazakh upland steppe)'),
+ (734, 'MongolianManchuriangrassland', '734 (Mongolian-Manchurian grassland)'),
+ (735, 'Ponticsteppe', '735 (Pontic steppe)'),
+ (736, 'SayanIntermontanesteppe', '736 (Sayan Intermontane steppe)'),
+ (737, 'SelengeOrkhonforeststeppe', '737 (Selenge-Orkhon forest steppe)'),
+ (738, 'SouthSiberianforeststeppe', '738 (South Siberian forest steppe)'),
+ (739, 'Syrianxericgrasslandsandshrublands', '739 (Syrian xeric grasslands and shrublands)'),
+ (740, 'TianShanfoothillaridsteppe', '740 (Tian Shan foothill arid steppe)'),
+ (741, 'Amurmeadowsteppe', '741 (Amur meadow steppe)'),
+ (742, 'BohaiSeasalinemeadow', '742 (Bohai Sea saline meadow)'),
+ (743, 'NenjiangRivergrassland', '743 (Nenjiang River grassland)'),
+ (744, 'NileDeltafloodedsavanna', '744 (Nile Delta flooded savanna)'),
+ (745, 'Saharanhalophytics', '745 (Saharan halophytics)'),
+ (746, 'SuiphunKhankameadowsandforestmeadows', '746 (Suiphun-Khanka meadows and forest meadows)'),
+ (747, 'TigrisEuphratesalluvialsaltmarsh', '747 (Tigris-Euphrates alluvial salt marsh)'),
+ (748, 'YellowSeasalinemeadow', '748 (Yellow Sea saline meadow)'),
+ (749, 'Altaialpinemeadowandtundra', '749 (Altai alpine meadow and tundra)'),
+ (750, 'CentralTibetanPlateaualpinesteppe', '750 (Central Tibetan Plateau alpine steppe)'),
+ (751, 'EasternHimalayanalpineshrubandmeadows', '751 (Eastern Himalayan alpine shrub and meadows)'),
+ (752, 'GhoratHazarajatalpinemeadow', '752 (Ghorat-Hazarajat alpine meadow)'),
+ (753, 'HinduKushalpinemeadow', '753 (Hindu Kush alpine meadow)'),
+ (754, 'KarakoramWestTibetanPlateaualpinesteppe', '754 (Karakoram-West Tibetan Plateau alpine steppe)'),
+ (755, 'KhangaiMountainsalpinemeadow', '755 (Khangai Mountains alpine meadow)'),
+ (756, 'KopetDagwoodlandsandforeststeppe', '756 (Kopet Dag woodlands and forest steppe)'),
+ (757, 'KuhRudandEasternIranmontanewoodlands', '757 (Kuh Rud and Eastern Iran montane woodlands)'),
+ (758, 'MediterraneanHighAtlasjunipersteppe', '758 (Mediterranean High Atlas juniper steppe)'),
+ (759, 'NorthTibetanPlateauKunlunMountainsalpinedesert', '759 (North Tibetan Plateau-Kunlun Mountains alpine desert)'),
+ (760, 'NorthwesternHimalayanalpineshrubandmeadows', '760 (Northwestern Himalayan alpine shrub and meadows)'),
+ (761, 'OrdosPlateausteppe', '761 (Ordos Plateau steppe)'),
+ (762, 'Pamiralpinedesertandtundra', '762 (Pamir alpine desert and tundra)'),
+ (763, 'QilianMountainssubalpinemeadows', '763 (Qilian Mountains subalpine meadows)'),
+ (764, 'Sayanalpinemeadowsandtundra', '764 (Sayan alpine meadows and tundra)'),
+ (765, 'SoutheastTibetshrublandsandmeadows', '765 (Southeast Tibet shrublands and meadows)'),
+ (766, 'SulaimanRangealpinemeadows', '766 (Sulaiman Range alpine meadows)'),
+ (767, 'TianShanmontanesteppeandmeadows', '767 (Tian Shan montane steppe and meadows)'),
+ (768, 'TibetanPlateaualpineshrublandsandmeadows', '768 (Tibetan Plateau alpine shrublands and meadows)'),
+ (769, 'WesternHimalayanalpineshrubandmeadows', '769 (Western Himalayan alpine shrub and meadows)'),
+ (770, 'YarlungZanboaridsteppe', '770 (Yarlung Zanbo arid steppe)'),
+ (771, 'CherskiiKolymamountaintundra', '771 (Cherskii-Kolyma mountain tundra)'),
+ (772, 'ChukchiPeninsulatundra', '772 (Chukchi Peninsula tundra)'),
+ (773, 'Kamchatkatundra', '773 (Kamchatka tundra)'),
+ (774, 'KolaPeninsulatundra', '774 (Kola Peninsula tundra)'),
+ (775, 'NortheastSiberiancoastaltundra', '775 (Northeast Siberian coastal tundra)'),
+ (776, 'NorthwestRussianNovayaZemlyatundra', '776 (Northwest Russian-Novaya Zemlya tundra)'),
+ (777, 'NovosibirskIslandsArcticdesert', '777 (Novosibirsk Islands Arctic desert)'),
+ (778, 'RussianArcticdesert', '778 (Russian Arctic desert)'),
+ (779, 'RussianBeringtundra', '779 (Russian Bering tundra)'),
+ (780, 'ScandinavianMontaneBirchforestandgrasslands', '780 (Scandinavian Montane Birch forest and grasslands)'),
+ (781, 'TaimyrCentralSiberiantundra', '781 (Taimyr-Central Siberian tundra)'),
+ (782, 'TransBaikalBaldMountaintundra', '782 (Trans-Baikal Bald Mountain tundra)'),
+ (783, 'WrangelIslandArcticdesert', '783 (Wrangel Island Arctic desert)'),
+ (784, 'YamalGydantundra', '784 (Yamal-Gydan tundra)'),
+ (785, 'AegeanandWesternTurkeysclerophyllousandmixedforests', '785 (Aegean and Western Turkey sclerophyllous and mixed forests)'),
+ (786, 'Anatolianconiferanddeciduousmixedforests', '786 (Anatolian conifer and deciduous mixed forests)'),
+ (787, 'CanaryIslandsdrywoodlandsandforests', '787 (Canary Islands dry woodlands and forests)'),
+ (788, 'Corsicanmontanebroadleafandmixedforests', '788 (Corsican montane broadleaf and mixed forests)'),
+ (789, 'CreteMediterraneanforests', '789 (Crete Mediterranean forests)'),
+ (790, 'CyprusMediterraneanforests', '790 (Cyprus Mediterranean forests)'),
+ (791, 'EasternMediterraneanconiferbroadleafforests', '791 (Eastern Mediterranean conifer-broadleaf forests)'),
+ (792, 'Iberianconiferforests', '792 (Iberian conifer forests)'),
+ (793, 'Iberiansclerophyllousandsemideciduousforests', '793 (Iberian sclerophyllous and semi-deciduous forests)'),
+ (794, 'Illyriandeciduousforests', '794 (Illyrian deciduous forests)'),
+ (795, 'Italiansclerophyllousandsemideciduousforests', '795 (Italian sclerophyllous and semi-deciduous forests)'),
+ (796, 'MediterraneanAcaciaArganiadrywoodlandsandsucculentthickets', '796 (Mediterranean Acacia-Argania dry woodlands and succulent thickets)'),
+ (797, 'Mediterraneandrywoodlandsandsteppe', '797 (Mediterranean dry woodlands and steppe)'),
+ (798, 'Mediterraneanwoodlandsandforests', '798 (Mediterranean woodlands and forests)'),
+ (799, 'NortheastSpainandSouthernFranceMediterraneanforests', '799 (Northeast Spain and Southern France Mediterranean forests)'),
+ (800, 'NorthwestIberianmontaneforests', '800 (Northwest Iberian montane forests)'),
+ (801, 'PindusMountainsmixedforests', '801 (Pindus Mountains mixed forests)'),
+ (802, 'SouthApenninemixedmontaneforests', '802 (South Apennine mixed montane forests)'),
+ (803, 'SoutheastIberianshrubsandwoodlands', '803 (Southeast Iberian shrubs and woodlands)'),
+ (804, 'SouthernAnatolianmontaneconiferanddeciduousforests', '804 (Southern Anatolian montane conifer and deciduous forests)'),
+ (805, 'SouthwestIberianMediterraneansclerophyllousandmixedforests', '805 (Southwest Iberian Mediterranean sclerophyllous and mixed forests)'),
+ (806, 'TyrrhenianAdriaticsclerophyllousandmixedforests', '806 (Tyrrhenian-Adriatic sclerophyllous and mixed forests)'),
+ (807, 'AfghanMountainssemidesert', '807 (Afghan Mountains semi-desert)'),
+ (808, 'AlashanPlateausemidesert', '808 (Alashan Plateau semi-desert)'),
+ (809, 'Arabiandesert', '809 (Arabian desert)'),
+ (810, 'Arabiansanddesert', '810 (Arabian sand desert)'),
+ (811, 'ArabianPersianGulfcoastalplaindesert', '811 (Arabian-Persian Gulf coastal plain desert)'),
+ (812, 'Azerbaijanshrubdesertandsteppe', '812 (Azerbaijan shrub desert and steppe)'),
+ (813, 'BadghyzandKarabilsemidesert', '813 (Badghyz and Karabil semi-desert)'),
+ (814, 'Baluchistanxericwoodlands', '814 (Baluchistan xeric woodlands)'),
+ (815, 'Caspianlowlanddesert', '815 (Caspian lowland desert)'),
+ (816, 'CentralAfghanMountainsxericwoodlands', '816 (Central Afghan Mountains xeric woodlands)'),
+ (817, 'CentralAsiannortherndesert', '817 (Central Asian northern desert)'),
+ (818, 'CentralAsianriparianwoodlands', '818 (Central Asian riparian woodlands)'),
+ (819, 'CentralAsiansoutherndesert', '819 (Central Asian southern desert)'),
+ (820, 'CentralPersiandesertbasins', '820 (Central Persian desert basins)'),
+ (821, 'EastArabianfogshrublandsandsanddesert', '821 (East Arabian fog shrublands and sand desert)'),
+ (822, 'EastSaharaDesert', '822 (East Sahara Desert)'),
+ (823, 'EastSaharanmontanexericwoodlands', '823 (East Saharan montane xeric woodlands)'),
+ (824, 'EasternGobidesertsteppe', '824 (Eastern Gobi desert steppe)'),
+ (825, 'GobiLakesValleydesertsteppe', '825 (Gobi Lakes Valley desert steppe)'),
+ (826, 'GreatLakesBasindesertsteppe', '826 (Great Lakes Basin desert steppe)'),
+ (827, 'JunggarBasinsemidesert', '827 (Junggar Basin semi-desert)'),
+ (828, 'Kazakhsemidesert', '828 (Kazakh semi-desert)'),
+ (829, 'KopetDagsemidesert', '829 (Kopet Dag semi-desert)'),
+ (830, 'Mesopotamianshrubdesert', '830 (Mesopotamian shrub desert)'),
+ (831, 'NorthArabiandesert', '831 (North Arabian desert)'),
+ (832, 'NorthArabianhighlandshrublands', '832 (North Arabian highland shrublands)'),
+ (833, 'NorthSaharanXericSteppeandWoodland', '833 (North Saharan Xeric Steppe and Woodland)'),
+ (834, 'Paropamisusxericwoodlands', '834 (Paropamisus xeric woodlands)'),
+ (835, 'QaidamBasinsemidesert', '835 (Qaidam Basin semi-desert)'),
+ (836, 'RedSeacoastaldesert', '836 (Red Sea coastal desert)'),
+ (837, 'RedSeaArabianDesertshrublands', '837 (Red Sea-Arabian Desert shrublands)'),
+ (838, 'RegistanNorthPakistansandydesert', '838 (Registan-North Pakistan sandy desert)'),
+ (839, 'SaharanAtlanticcoastaldesert', '839 (Saharan Atlantic coastal desert)'),
+ (840, 'SouthArabianplainsandplateaudesert', '840 (South Arabian plains and plateau desert)'),
+ (841, 'SouthIranNuboSindiandesertandsemidesert', '841 (South Iran Nubo-Sindian desert and semi-desert)'),
+ (842, 'SouthSaharadesert', '842 (South Sahara desert)'),
+ (843, 'Taklimakandesert', '843 (Taklimakan desert)'),
+ (844, 'TibestiJebelUweinatmontanexericwoodlands', '844 (Tibesti-Jebel Uweinat montane xeric woodlands)'),
+ (845, 'WestSaharadesert', '845 (West Sahara desert)'),
+ (846, 'WestSaharanmontanexericwoodlands', '846 (West Saharan montane xeric woodlands)');
+
+-- Result Types
+
+CREATE TABLE IF NOT EXISTS RT_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT rt_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO RT_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'String', 'str'),
+ (1, 'Integer', 'int'),
+ (2, 'Float', 'float');
+
+-- Data
+-- ====
+
+-- Data Flags
+
+CREATE TABLE IF NOT EXISTS DF_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT df_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO DF_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ ( 0, 'OKValidatedVerified', 'OK validated verified'),
+ ( 1, 'OKValidatedQCPassed', 'OK validated QC passed'),
+ ( 2, 'OKValidatedModified', 'OK validated modified'),
+ ( 3, 'OKPreliminaryVerified', 'OK preliminary verified'),
+ ( 4, 'OKPreliminaryQCPassed', 'OK preliminary QC passed'),
+ ( 5, 'OKPreliminaryModified', 'OK preliminary modified'),
+ ( 6, 'OKEstimated', 'OK estimated'),
+ ( 7, 'OKPreliminaryNotChecked', 'OK preliminary not checked'),
+ ( 10, 'QuestionableValidatedConfirmed', 'questionable validated confirmed'),
+ ( 11, 'QuestionableValidatedUnconfirmed', 'questionable validated unconfirmed'),
+ ( 12, 'QuestionableValidatedFlagged', 'questionable validated flagged'),
+ ( 13, 'QuestionablePreliminaryConfirmed', 'questionable preliminary confirmed'),
+ ( 14, 'QuestionablePreliminaryUnconfirmed', 'questionable preliminary unconfirmed'),
+ ( 15, 'QuestionablePreliminaryFlagged', 'questionable preliminary flagged'),
+ ( 16, 'QuestionablePreliminaryNotChecked', 'questionable preliminary not checked'),
+ ( 20, 'ErroneousValidatedConfirmed', 'erroneous validated confirmed'),
+ ( 21, 'ErroneousValidatedUnconfirmed', 'erroneous validated unconfirmed'),
+ ( 22, 'ErroneousValidatedFlagged1', 'erroneous validated flagged (1)'),
+ ( 23, 'ErroneousValidatedFlagged2', 'erroneous validated flagged (2)'),
+ ( 24, 'ErroneousPreliminaryConfirmed', 'erroneous preliminary confirmed'),
+ ( 25, 'ErroneousPreliminaryUnconfirmed', 'erroneous preliminary unconfirmed'),
+ ( 26, 'ErroneousPreliminaryFlagged1', 'erroneous preliminary flagged (1)'),
+ ( 27, 'ErroneousPreliminaryFlagged2', 'erroneous preliminary flagged (2)'),
+ ( 28, 'ErroneousPreliminaryNotChecked', 'erroneous preliminary not checked'),
+ ( 90, 'MissingValue', 'missing value'),
+ ( 91, 'UnknownQualityStatus', 'unknown quality status'),
+ (100, 'AllOK', 'all OK'),
+ (101, 'ValidatedOK', 'validated OK'),
+ (102, 'PreliminaryOK', 'preliminary OK'),
+ (103, 'NotModifiedOK', 'not modified OK'),
+ (104, 'ModifiedOK', 'modified OK'),
+ (110, 'AllQuestionable', 'all questionable'),
+ (111, 'ValidatedQuestionable', 'validated questionable'),
+ (112, 'PreliminaryQuestionable', 'preliminary questionable'),
+ (120, 'AllErroneous', 'all erroneous'),
+ (121, 'ValidatedErroneous', 'validated erroneous'),
+ (122, 'PreliminaryErroneous', 'preliminary erroneous'),
+ (130, 'AllQuestionableOrErroneous', 'all questionable or erroneous'),
+ (131, 'ValidatedQuestionableOrErroneous', 'validated questionable or erroneous'),
+ (132, 'PreliminaryQuestionableOrErroneous', 'preliminary questionable or erroneous'),
+ (140, 'NotChecked', 'not checked');
+
+
+-- Countries
+-- =========
+
+-- Country names
+
+CREATE TABLE IF NOT EXISTS CN_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT cn_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO CN_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (-1, 'N/A', 'N/A'),
+ (0, 'AF', 'Afghanistan'),
+ (1, 'AX', 'Åland Islands'),
+ (2, 'AL', 'Albania'),
+ (3, 'DZ', 'Algeria'),
+ (4, 'AS', 'American Samoa'),
+ (5, 'AD', 'Andorra'),
+ (6, 'AO', 'Angola'),
+ (7, 'AI', 'Anguilla'),
+ (8, 'AQ', 'Antarctica'),
+ (9, 'AG', 'Antigua and Barbuda'),
+ (10, 'AR', 'Argentina'),
+ (11, 'AM', 'Armenia'),
+ (12, 'AW', 'Aruba'),
+ (13, 'AU', 'Australia'),
+ (14, 'AT', 'Austria'),
+ (15, 'AZ', 'Azerbaijan'),
+ (16, 'BS', 'Bahamas'),
+ (17, 'BH', 'Bahrain'),
+ (18, 'BD', 'Bangladesh'),
+ (19, 'BB', 'Barbados'),
+ (20, 'BY', 'Belarus'),
+ (21, 'BE', 'Belgium'),
+ (22, 'BZ', 'Belize'),
+ (23, 'BJ', 'Benin'),
+ (24, 'BM', 'Bermuda'),
+ (25, 'BT', 'Bhutan'),
+ (26, 'BO', 'Plurinational State of Bolivia'),
+ (27, 'BQ', 'Bonaire, Sint Eustatius and Saba'),
+ (28, 'BA', 'Bosnia and Herzegovina'),
+ (29, 'BW', 'Botswana'),
+ (30, 'BV', 'Bouvet Island'),
+ (31, 'BR', 'Brazil'),
+ (32, 'IO', 'British Indian Ocean Territory'),
+ (33, 'UM', 'United States Minor Outlying Islands'),
+ (34, 'VG', 'British Virgin Islands'),
+ (35, 'VI', 'U.S. Virgin Islands'),
+ (36, 'BN', 'Brunei Darussalam'),
+ (37, 'BG', 'Bulgaria'),
+ (38, 'BF', 'Burkina Faso'),
+ (39, 'BI', 'Burundi'),
+ (40, 'KH', 'Cambodia'),
+ (41, 'CM', 'Cameroon'),
+ (42, 'CA', 'Canada'),
+ (43, 'CV', 'Cabo Verde'),
+ (44, 'KY', 'Cayman Islands'),
+ (45, 'CF', 'Central African Republic'),
+ (46, 'TD', 'Chad'),
+ (47, 'CL', 'Chile'),
+ (48, 'CN', 'China'),
+ (49, 'CX', 'Christmas Island'),
+ (50, 'CC', 'Cocos (Keeling) Islands'),
+ (51, 'CO', 'Colombia'),
+ (52, 'KM', 'Comoros'),
+ (53, 'CG', 'Congo'),
+ (54, 'CD', 'Democratic Republic of the Congo'),
+ (55, 'CK', 'Cook Islands'),
+ (56, 'CR', 'Costa Rica'),
+ (57, 'HR', 'Croatia'),
+ (58, 'CU', 'Cuba'),
+ (59, 'CW', 'Curaçao'),
+ (60, 'CY', 'Cyprus'),
+ (61, 'CZ', 'Czech Republic'),
+ (62, 'DK', 'Denmark'),
+ (63, 'DJ', 'Djibouti'),
+ (64, 'DM', 'Dominica'),
+ (65, 'DO', 'Dominican Republic'),
+ (66, 'EC', 'Ecuador'),
+ (67, 'EG', 'Egypt'),
+ (68, 'SV', 'El Salvador'),
+ (69, 'GQ', 'Equatorial Guinea'),
+ (70, 'ER', 'Eritrea'),
+ (71, 'EE', 'Estonia'),
+ (72, 'ET', 'Ethiopia'),
+ (73, 'FK', 'Falkland Islands (Malvinas)'),
+ (74, 'FO', 'Faroe Islands'),
+ (75, 'FJ', 'Fiji'),
+ (76, 'FI', 'Finland'),
+ (77, 'FR', 'France'),
+ (78, 'GF', 'French Guiana'),
+ (79, 'PF', 'French Polynesia'),
+ (80, 'TF', 'French Southern Territories'),
+ (81, 'GA', 'Gabon'),
+ (82, 'GM', 'Gambia'),
+ (83, 'GE', 'Georgia'),
+ (84, 'DE', 'Germany'),
+ (85, 'GH', 'Ghana'),
+ (86, 'GI', 'Gibraltar'),
+ (87, 'GR', 'Greece'),
+ (88, 'GL', 'Greenland'),
+ (89, 'GD', 'Grenada'),
+ (90, 'GP', 'Guadeloupe'),
+ (91, 'GU', 'Guam'),
+ (92, 'GT', 'Guatemala'),
+ (93, 'GG', 'Guernsey'),
+ (94, 'GN', 'Guinea'),
+ (95, 'GW', 'Guinea-Bissau'),
+ (96, 'GY', 'Guyana'),
+ (97, 'HT', 'Haiti'),
+ (98, 'HM', 'Heard Island and McDonald Islands'),
+ (99, 'VA', 'Holy See'),
+ (100, 'HN', 'Honduras'),
+ (101, 'HK', 'Hong Kong'),
+ (102, 'HU', 'Hungary'),
+ (103, 'IS', 'Iceland'),
+ (104, 'IN', 'India'),
+ (105, 'ID', 'Indonesia'),
+ (106, 'CI', 'Côte d''Ivoire'),
+ (107, 'IR', 'Islamic Republic of Iran'),
+ (108, 'IQ', 'Iraq'),
+ (109, 'IE', 'Ireland'),
+ (110, 'IM', 'Isle of Man'),
+ (111, 'IL', 'Israel'),
+ (112, 'IT', 'Italy'),
+ (113, 'JM', 'Jamaica'),
+ (114, 'JP', 'Japan'),
+ (115, 'JE', 'Jersey'),
+ (116, 'JO', 'Jordan'),
+ (117, 'KZ', 'Kazakhstan'),
+ (118, 'KE', 'Kenya'),
+ (119, 'KI', 'Kiribati'),
+ (120, 'KW', 'Kuwait'),
+ (121, 'KG', 'Kyrgyzstan'),
+ (122, 'LA', 'Lao People''s Democratic Republic'),
+ (123, 'LV', 'Latvia'),
+ (124, 'LB', 'Lebanon'),
+ (125, 'LS', 'Lesotho'),
+ (126, 'LR', 'Liberia'),
+ (127, 'LY', 'Libya'),
+ (128, 'LI', 'Liechtenstein'),
+ (129, 'LT', 'Lithuania'),
+ (130, 'LU', 'Luxembourg'),
+ (131, 'MO', 'Macao'),
+ (132, 'MK', 'North Macedonia'),
+ (133, 'MG', 'Madagascar'),
+ (134, 'MW', 'Malawi'),
+ (135, 'MY', 'Malaysia'),
+ (136, 'MV', 'Maldives'),
+ (137, 'ML', 'Mali'),
+ (138, 'MT', 'Malta'),
+ (139, 'MH', 'Marshall Islands'),
+ (140, 'MQ', 'Martinique'),
+ (141, 'MR', 'Mauritania'),
+ (142, 'MU', 'Mauritius'),
+ (143, 'YT', 'Mayotte'),
+ (144, 'MX', 'Mexico'),
+ (145, 'FM', 'Federated States of Micronesia'),
+ (146, 'MD', 'Republic of Moldova'),
+ (147, 'MC', 'Monaco'),
+ (148, 'MN', 'Mongolia'),
+ (149, 'ME', 'Montenegro'),
+ (150, 'MS', 'Montserrat'),
+ (151, 'MA', 'Morocco'),
+ (152, 'MZ', 'Mozambique'),
+ (153, 'MM', 'Myanmar'),
+ (154, 'NA', 'Namibia'),
+ (155, 'NR', 'Nauru'),
+ (156, 'NP', 'Nepal'),
+ (157, 'NL', 'Netherlands'),
+ (158, 'NC', 'New Caledonia'),
+ (159, 'NZ', 'New Zealand'),
+ (160, 'NI', 'Nicaragua'),
+ (161, 'NE', 'Niger'),
+ (162, 'NG', 'Nigeria'),
+ (163, 'NU', 'Niue'),
+ (164, 'NF', 'Norfolk Island'),
+ (165, 'KP', 'Democratic People''s Republic of Korea'),
+ (166, 'MP', 'Northern Mariana Islands'),
+ (167, 'NO', 'Norway'),
+ (168, 'OM', 'Oman'),
+ (169, 'PK', 'Pakistan'),
+ (170, 'PW', 'Palau'),
+ (171, 'PS', 'State of Palestine'),
+ (172, 'PA', 'Panama'),
+ (173, 'PG', 'Papua New Guinea'),
+ (174, 'PY', 'Paraguay'),
+ (175, 'PE', 'Peru'),
+ (176, 'PH', 'Philippines'),
+ (177, 'PN', 'Pitcairn'),
+ (178, 'PL', 'Poland'),
+ (179, 'PT', 'Portugal'),
+ (180, 'PR', 'Puerto Rico'),
+ (181, 'QA', 'Qatar'),
+ (182, 'XK', 'Republic of Kosovo'),
+ (183, 'RE', 'Réunion'),
+ (184, 'RO', 'Romania'),
+ (185, 'RU', 'Russian Federation'),
+ (186, 'RW', 'Rwanda'),
+ (187, 'BL', 'Saint Barthélemy'),
+ (188, 'SH', 'Saint Helena, Ascension and Tristan da Cunha'),
+ (189, 'KN', 'Saint Kitts and Nevis'),
+ (190, 'LC', 'Saint Lucia'),
+ (191, 'MF', 'Saint Martin (French part)'),
+ (192, 'PM', 'Saint Pierre and Miquelon'),
+ (193, 'VC', 'Saint Vincent and the Grenadines'),
+ (194, 'WS', 'Samoa'),
+ (195, 'SM', 'San Marino'),
+ (196, 'ST', 'Sao Tome and Principe'),
+ (197, 'SA', 'Saudi Arabia'),
+ (198, 'SN', 'Senegal'),
+ (199, 'RS', 'Serbia'),
+ (200, 'SC', 'Seychelles'),
+ (201, 'SL', 'Sierra Leone'),
+ (202, 'SG', 'Singapore'),
+ (203, 'SX', 'Sint Maarten (Dutch part)'),
+ (204, 'SK', 'Slovakia'),
+ (205, 'SI', 'Slovenia'),
+ (206, 'SB', 'Solomon Islands'),
+ (207, 'SO', 'Somalia'),
+ (208, 'ZA', 'South Africa'),
+ (209, 'GS', 'South Georgia and the South Sandwich Islands'),
+ (210, 'KR', 'Republic of Korea'),
+ (211, 'SS', 'South Sudan'),
+ (212, 'ES', 'Spain'),
+ (213, 'LK', 'Sri Lanka'),
+ (214, 'SD', 'Sudan'),
+ (215, 'SR', 'Suriname'),
+ (216, 'SJ', 'Svalbard and Jan Mayen'),
+ (217, 'SZ', 'Swaziland'),
+ (218, 'SE', 'Sweden'),
+ (219, 'CH', 'Switzerland'),
+ (220, 'SY', 'Syrian Arab Republic'),
+ (221, 'TW', 'Taiwan'),
+ (222, 'TJ', 'Tajikistan'),
+ (223, 'TZ', 'United Republic of Tanzania'),
+ (224, 'TH', 'Thailand'),
+ (225, 'TL', 'Timor-Leste'),
+ (226, 'TG', 'Togo'),
+ (227, 'TK', 'Tokelau'),
+ (228, 'TO', 'Tonga'),
+ (229, 'TT', 'Trinidad and Tobago'),
+ (230, 'TN', 'Tunisia'),
+ (231, 'TR', 'Turkey'),
+ (232, 'TM', 'Turkmenistan'),
+ (233, 'TC', 'Turks and Caicos Islands'),
+ (234, 'TV', 'Tuvalu'),
+ (235, 'UG', 'Uganda'),
+ (236, 'UA', 'Ukraine'),
+ (237, 'AE', 'United Arab Emirates'),
+ (238, 'GB', 'United Kingdom of Great Britain and Northern Ireland'),
+ (239, 'US', 'United States of America'),
+ (240, 'UY', 'Uruguay'),
+ (241, 'UZ', 'Uzbekistan'),
+ (242, 'VU', 'Vanuatu'),
+ (243, 'VE', 'Bolivarian Republic of Venezuela'),
+ (244, 'VN', 'Viet Nam'),
+ (245, 'WF', 'Wallis and Futuna'),
+ (246, 'EH', 'Western Sahara'),
+ (247, 'YE', 'Yemen'),
+ (248, 'ZM', 'Zambia'),
+ (249, 'ZW', 'Zimbabwe');
+
+-- Timezones (from pytz)
+-- =====================
+
+CREATE TABLE IF NOT EXISTS TZ_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT tz_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO TZ_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (-1, 'N/A', 'N/A'),
+ (0, 'Africa/Abidjan', 'Africa/Abidjan'),
+ (1, 'Africa/Accra', 'Africa/Accra'),
+ (2, 'Africa/Addis_Ababa', 'Africa/Addis_Ababa'),
+ (3, 'Africa/Algiers', 'Africa/Algiers'),
+ (4, 'Africa/Asmara', 'Africa/Asmara'),
+ (5, 'Africa/Asmera', 'Africa/Asmera'),
+ (6, 'Africa/Bamako', 'Africa/Bamako'),
+ (7, 'Africa/Bangui', 'Africa/Bangui'),
+ (8, 'Africa/Banjul', 'Africa/Banjul'),
+ (9, 'Africa/Bissau', 'Africa/Bissau'),
+ (10, 'Africa/Blantyre', 'Africa/Blantyre'),
+ (11, 'Africa/Brazzaville', 'Africa/Brazzaville'),
+ (12, 'Africa/Bujumbura', 'Africa/Bujumbura'),
+ (13, 'Africa/Cairo', 'Africa/Cairo'),
+ (14, 'Africa/Casablanca', 'Africa/Casablanca'),
+ (15, 'Africa/Ceuta', 'Africa/Ceuta'),
+ (16, 'Africa/Conakry', 'Africa/Conakry'),
+ (17, 'Africa/Dakar', 'Africa/Dakar'),
+ (18, 'Africa/Dar_es_Salaam', 'Africa/Dar_es_Salaam'),
+ (19, 'Africa/Djibouti', 'Africa/Djibouti'),
+ (20, 'Africa/Douala', 'Africa/Douala'),
+ (21, 'Africa/El_Aaiun', 'Africa/El_Aaiun'),
+ (22, 'Africa/Freetown', 'Africa/Freetown'),
+ (23, 'Africa/Gaborone', 'Africa/Gaborone'),
+ (24, 'Africa/Harare', 'Africa/Harare'),
+ (25, 'Africa/Johannesburg', 'Africa/Johannesburg'),
+ (26, 'Africa/Juba', 'Africa/Juba'),
+ (27, 'Africa/Kampala', 'Africa/Kampala'),
+ (28, 'Africa/Khartoum', 'Africa/Khartoum'),
+ (29, 'Africa/Kigali', 'Africa/Kigali'),
+ (30, 'Africa/Kinshasa', 'Africa/Kinshasa'),
+ (31, 'Africa/Lagos', 'Africa/Lagos'),
+ (32, 'Africa/Libreville', 'Africa/Libreville'),
+ (33, 'Africa/Lome', 'Africa/Lome'),
+ (34, 'Africa/Luanda', 'Africa/Luanda'),
+ (35, 'Africa/Lubumbashi', 'Africa/Lubumbashi'),
+ (36, 'Africa/Lusaka', 'Africa/Lusaka'),
+ (37, 'Africa/Malabo', 'Africa/Malabo'),
+ (38, 'Africa/Maputo', 'Africa/Maputo'),
+ (39, 'Africa/Maseru', 'Africa/Maseru'),
+ (40, 'Africa/Mbabane', 'Africa/Mbabane'),
+ (41, 'Africa/Mogadishu', 'Africa/Mogadishu'),
+ (42, 'Africa/Monrovia', 'Africa/Monrovia'),
+ (43, 'Africa/Nairobi', 'Africa/Nairobi'),
+ (44, 'Africa/Ndjamena', 'Africa/Ndjamena'),
+ (45, 'Africa/Niamey', 'Africa/Niamey'),
+ (46, 'Africa/Nouakchott', 'Africa/Nouakchott'),
+ (47, 'Africa/Ouagadougou', 'Africa/Ouagadougou'),
+ (48, 'Africa/Porto-Novo', 'Africa/Porto-Novo'),
+ (49, 'Africa/Sao_Tome', 'Africa/Sao_Tome'),
+ (50, 'Africa/Timbuktu', 'Africa/Timbuktu'),
+ (51, 'Africa/Tripoli', 'Africa/Tripoli'),
+ (52, 'Africa/Tunis', 'Africa/Tunis'),
+ (53, 'Africa/Windhoek', 'Africa/Windhoek'),
+ (54, 'America/Adak', 'America/Adak'),
+ (55, 'America/Anchorage', 'America/Anchorage'),
+ (56, 'America/Anguilla', 'America/Anguilla'),
+ (57, 'America/Antigua', 'America/Antigua'),
+ (58, 'America/Araguaina', 'America/Araguaina'),
+ (59, 'America/Argentina/Buenos_Aires', 'America/Argentina/Buenos_Aires'),
+ (60, 'America/Argentina/Catamarca', 'America/Argentina/Catamarca'),
+ (61, 'America/Argentina/ComodRivadavia', 'America/Argentina/ComodRivadavia'),
+ (62, 'America/Argentina/Cordoba', 'America/Argentina/Cordoba'),
+ (63, 'America/Argentina/Jujuy', 'America/Argentina/Jujuy'),
+ (64, 'America/Argentina/La_Rioja', 'America/Argentina/La_Rioja'),
+ (65, 'America/Argentina/Mendoza', 'America/Argentina/Mendoza'),
+ (66, 'America/Argentina/Rio_Gallegos', 'America/Argentina/Rio_Gallegos'),
+ (67, 'America/Argentina/Salta', 'America/Argentina/Salta'),
+ (68, 'America/Argentina/San_Juan', 'America/Argentina/San_Juan'),
+ (69, 'America/Argentina/San_Luis', 'America/Argentina/San_Luis'),
+ (70, 'America/Argentina/Tucuman', 'America/Argentina/Tucuman'),
+ (71, 'America/Argentina/Ushuaia', 'America/Argentina/Ushuaia'),
+ (72, 'America/Aruba', 'America/Aruba'),
+ (73, 'America/Asuncion', 'America/Asuncion'),
+ (74, 'America/Atikokan', 'America/Atikokan'),
+ (75, 'America/Atka', 'America/Atka'),
+ (76, 'America/Bahia', 'America/Bahia'),
+ (77, 'America/Bahia_Banderas', 'America/Bahia_Banderas'),
+ (78, 'America/Barbados', 'America/Barbados'),
+ (79, 'America/Belem', 'America/Belem'),
+ (80, 'America/Belize', 'America/Belize'),
+ (81, 'America/Blanc-Sablon', 'America/Blanc-Sablon'),
+ (82, 'America/Boa_Vista', 'America/Boa_Vista'),
+ (83, 'America/Bogota', 'America/Bogota'),
+ (84, 'America/Boise', 'America/Boise'),
+ (85, 'America/Buenos_Aires', 'America/Buenos_Aires'),
+ (86, 'America/Cambridge_Bay', 'America/Cambridge_Bay'),
+ (87, 'America/Campo_Grande', 'America/Campo_Grande'),
+ (88, 'America/Cancun', 'America/Cancun'),
+ (89, 'America/Caracas', 'America/Caracas'),
+ (90, 'America/Catamarca', 'America/Catamarca'),
+ (91, 'America/Cayenne', 'America/Cayenne'),
+ (92, 'America/Cayman', 'America/Cayman'),
+ (93, 'America/Chicago', 'America/Chicago'),
+ (94, 'America/Chihuahua', 'America/Chihuahua'),
+ (95, 'America/Coral_Harbour', 'America/Coral_Harbour'),
+ (96, 'America/Cordoba', 'America/Cordoba'),
+ (97, 'America/Costa_Rica', 'America/Costa_Rica'),
+ (98, 'America/Creston', 'America/Creston'),
+ (99, 'America/Cuiaba', 'America/Cuiaba'),
+ (100, 'America/Curacao', 'America/Curacao'),
+ (101, 'America/Danmarkshavn', 'America/Danmarkshavn'),
+ (102, 'America/Dawson', 'America/Dawson'),
+ (103, 'America/Dawson_Creek', 'America/Dawson_Creek'),
+ (104, 'America/Denver', 'America/Denver'),
+ (105, 'America/Detroit', 'America/Detroit'),
+ (106, 'America/Dominica', 'America/Dominica'),
+ (107, 'America/Edmonton', 'America/Edmonton'),
+ (108, 'America/Eirunepe', 'America/Eirunepe'),
+ (109, 'America/El_Salvador', 'America/El_Salvador'),
+ (110, 'America/Ensenada', 'America/Ensenada'),
+ (111, 'America/Fort_Nelson', 'America/Fort_Nelson'),
+ (112, 'America/Fort_Wayne', 'America/Fort_Wayne'),
+ (113, 'America/Fortaleza', 'America/Fortaleza'),
+ (114, 'America/Glace_Bay', 'America/Glace_Bay'),
+ (115, 'America/Godthab', 'America/Godthab'),
+ (116, 'America/Goose_Bay', 'America/Goose_Bay'),
+ (117, 'America/Grand_Turk', 'America/Grand_Turk'),
+ (118, 'America/Grenada', 'America/Grenada'),
+ (119, 'America/Guadeloupe', 'America/Guadeloupe'),
+ (120, 'America/Guatemala', 'America/Guatemala'),
+ (121, 'America/Guayaquil', 'America/Guayaquil'),
+ (122, 'America/Guyana', 'America/Guyana'),
+ (123, 'America/Halifax', 'America/Halifax'),
+ (124, 'America/Havana', 'America/Havana'),
+ (125, 'America/Hermosillo', 'America/Hermosillo'),
+ (126, 'America/Indiana/Indianapolis', 'America/Indiana/Indianapolis'),
+ (127, 'America/Indiana/Knox', 'America/Indiana/Knox'),
+ (128, 'America/Indiana/Marengo', 'America/Indiana/Marengo'),
+ (129, 'America/Indiana/Petersburg', 'America/Indiana/Petersburg'),
+ (130, 'America/Indiana/Tell_City', 'America/Indiana/Tell_City'),
+ (131, 'America/Indiana/Vevay', 'America/Indiana/Vevay'),
+ (132, 'America/Indiana/Vincennes', 'America/Indiana/Vincennes'),
+ (133, 'America/Indiana/Winamac', 'America/Indiana/Winamac'),
+ (134, 'America/Indianapolis', 'America/Indianapolis'),
+ (135, 'America/Inuvik', 'America/Inuvik'),
+ (136, 'America/Iqaluit', 'America/Iqaluit'),
+ (137, 'America/Jamaica', 'America/Jamaica'),
+ (138, 'America/Jujuy', 'America/Jujuy'),
+ (139, 'America/Juneau', 'America/Juneau'),
+ (140, 'America/Kentucky/Louisville', 'America/Kentucky/Louisville'),
+ (141, 'America/Kentucky/Monticello', 'America/Kentucky/Monticello'),
+ (142, 'America/Knox_IN', 'America/Knox_IN'),
+ (143, 'America/Kralendijk', 'America/Kralendijk'),
+ (144, 'America/La_Paz', 'America/La_Paz'),
+ (145, 'America/Lima', 'America/Lima'),
+ (146, 'America/Los_Angeles', 'America/Los_Angeles'),
+ (147, 'America/Louisville', 'America/Louisville'),
+ (148, 'America/Lower_Princes', 'America/Lower_Princes'),
+ (149, 'America/Maceio', 'America/Maceio'),
+ (150, 'America/Managua', 'America/Managua'),
+ (151, 'America/Manaus', 'America/Manaus'),
+ (152, 'America/Marigot', 'America/Marigot'),
+ (153, 'America/Martinique', 'America/Martinique'),
+ (154, 'America/Matamoros', 'America/Matamoros'),
+ (155, 'America/Mazatlan', 'America/Mazatlan'),
+ (156, 'America/Mendoza', 'America/Mendoza'),
+ (157, 'America/Menominee', 'America/Menominee'),
+ (158, 'America/Merida', 'America/Merida'),
+ (159, 'America/Metlakatla', 'America/Metlakatla'),
+ (160, 'America/Mexico_City', 'America/Mexico_City'),
+ (161, 'America/Miquelon', 'America/Miquelon'),
+ (162, 'America/Moncton', 'America/Moncton'),
+ (163, 'America/Monterrey', 'America/Monterrey'),
+ (164, 'America/Montevideo', 'America/Montevideo'),
+ (165, 'America/Montreal', 'America/Montreal'),
+ (166, 'America/Montserrat', 'America/Montserrat'),
+ (167, 'America/Nassau', 'America/Nassau'),
+ (168, 'America/New_York', 'America/New_York'),
+ (169, 'America/Nipigon', 'America/Nipigon'),
+ (170, 'America/Nome', 'America/Nome'),
+ (171, 'America/Noronha', 'America/Noronha'),
+ (172, 'America/North_Dakota/Beulah', 'America/North_Dakota/Beulah'),
+ (173, 'America/North_Dakota/Center', 'America/North_Dakota/Center'),
+ (174, 'America/North_Dakota/New_Salem', 'America/North_Dakota/New_Salem'),
+ (175, 'America/Nuuk', 'America/Nuuk'),
+ (176, 'America/Ojinaga', 'America/Ojinaga'),
+ (177, 'America/Panama', 'America/Panama'),
+ (178, 'America/Pangnirtung', 'America/Pangnirtung'),
+ (179, 'America/Paramaribo', 'America/Paramaribo'),
+ (180, 'America/Phoenix', 'America/Phoenix'),
+ (181, 'America/Port-au-Prince', 'America/Port-au-Prince'),
+ (182, 'America/Port_of_Spain', 'America/Port_of_Spain'),
+ (183, 'America/Porto_Acre', 'America/Porto_Acre'),
+ (184, 'America/Porto_Velho', 'America/Porto_Velho'),
+ (185, 'America/Puerto_Rico', 'America/Puerto_Rico'),
+ (186, 'America/Punta_Arenas', 'America/Punta_Arenas'),
+ (187, 'America/Rainy_River', 'America/Rainy_River'),
+ (188, 'America/Rankin_Inlet', 'America/Rankin_Inlet'),
+ (189, 'America/Recife', 'America/Recife'),
+ (190, 'America/Regina', 'America/Regina'),
+ (191, 'America/Resolute', 'America/Resolute'),
+ (192, 'America/Rio_Branco', 'America/Rio_Branco'),
+ (193, 'America/Rosario', 'America/Rosario'),
+ (194, 'America/Santa_Isabel', 'America/Santa_Isabel'),
+ (195, 'America/Santarem', 'America/Santarem'),
+ (196, 'America/Santiago', 'America/Santiago'),
+ (197, 'America/Santo_Domingo', 'America/Santo_Domingo'),
+ (198, 'America/Sao_Paulo', 'America/Sao_Paulo'),
+ (199, 'America/Scoresbysund', 'America/Scoresbysund'),
+ (200, 'America/Shiprock', 'America/Shiprock'),
+ (201, 'America/Sitka', 'America/Sitka'),
+ (202, 'America/St_Barthelemy', 'America/St_Barthelemy'),
+ (203, 'America/St_Johns', 'America/St_Johns'),
+ (204, 'America/St_Kitts', 'America/St_Kitts'),
+ (205, 'America/St_Lucia', 'America/St_Lucia'),
+ (206, 'America/St_Thomas', 'America/St_Thomas'),
+ (207, 'America/St_Vincent', 'America/St_Vincent'),
+ (208, 'America/Swift_Current', 'America/Swift_Current'),
+ (209, 'America/Tegucigalpa', 'America/Tegucigalpa'),
+ (210, 'America/Thule', 'America/Thule'),
+ (211, 'America/Thunder_Bay', 'America/Thunder_Bay'),
+ (212, 'America/Tijuana', 'America/Tijuana'),
+ (213, 'America/Toronto', 'America/Toronto'),
+ (214, 'America/Tortola', 'America/Tortola'),
+ (215, 'America/Vancouver', 'America/Vancouver'),
+ (216, 'America/Virgin', 'America/Virgin'),
+ (217, 'America/Whitehorse', 'America/Whitehorse'),
+ (218, 'America/Winnipeg', 'America/Winnipeg'),
+ (219, 'America/Yakutat', 'America/Yakutat'),
+ (220, 'America/Yellowknife', 'America/Yellowknife'),
+ (221, 'Antarctica/Casey', 'Antarctica/Casey'),
+ (222, 'Antarctica/Davis', 'Antarctica/Davis'),
+ (223, 'Antarctica/DumontDUrville', 'Antarctica/DumontDUrville'),
+ (224, 'Antarctica/Macquarie', 'Antarctica/Macquarie'),
+ (225, 'Antarctica/Mawson', 'Antarctica/Mawson'),
+ (226, 'Antarctica/McMurdo', 'Antarctica/McMurdo'),
+ (227, 'Antarctica/Palmer', 'Antarctica/Palmer'),
+ (228, 'Antarctica/Rothera', 'Antarctica/Rothera'),
+ (229, 'Antarctica/South_Pole', 'Antarctica/South_Pole'),
+ (230, 'Antarctica/Syowa', 'Antarctica/Syowa'),
+ (231, 'Antarctica/Troll', 'Antarctica/Troll'),
+ (232, 'Antarctica/Vostok', 'Antarctica/Vostok'),
+ (233, 'Arctic/Longyearbyen', 'Arctic/Longyearbyen'),
+ (234, 'Asia/Aden', 'Asia/Aden'),
+ (235, 'Asia/Almaty', 'Asia/Almaty'),
+ (236, 'Asia/Amman', 'Asia/Amman'),
+ (237, 'Asia/Anadyr', 'Asia/Anadyr'),
+ (238, 'Asia/Aqtau', 'Asia/Aqtau'),
+ (239, 'Asia/Aqtobe', 'Asia/Aqtobe'),
+ (240, 'Asia/Ashgabat', 'Asia/Ashgabat'),
+ (241, 'Asia/Ashkhabad', 'Asia/Ashkhabad'),
+ (242, 'Asia/Atyrau', 'Asia/Atyrau'),
+ (243, 'Asia/Baghdad', 'Asia/Baghdad'),
+ (244, 'Asia/Bahrain', 'Asia/Bahrain'),
+ (245, 'Asia/Baku', 'Asia/Baku'),
+ (246, 'Asia/Bangkok', 'Asia/Bangkok'),
+ (247, 'Asia/Barnaul', 'Asia/Barnaul'),
+ (248, 'Asia/Beirut', 'Asia/Beirut'),
+ (249, 'Asia/Bishkek', 'Asia/Bishkek'),
+ (250, 'Asia/Brunei', 'Asia/Brunei'),
+ (251, 'Asia/Calcutta', 'Asia/Calcutta'),
+ (252, 'Asia/Chita', 'Asia/Chita'),
+ (253, 'Asia/Choibalsan', 'Asia/Choibalsan'),
+ (254, 'Asia/Chongqing', 'Asia/Chongqing'),
+ (255, 'Asia/Chungking', 'Asia/Chungking'),
+ (256, 'Asia/Colombo', 'Asia/Colombo'),
+ (257, 'Asia/Dacca', 'Asia/Dacca'),
+ (258, 'Asia/Damascus', 'Asia/Damascus'),
+ (259, 'Asia/Dhaka', 'Asia/Dhaka'),
+ (260, 'Asia/Dili', 'Asia/Dili'),
+ (261, 'Asia/Dubai', 'Asia/Dubai'),
+ (262, 'Asia/Dushanbe', 'Asia/Dushanbe'),
+ (263, 'Asia/Famagusta', 'Asia/Famagusta'),
+ (264, 'Asia/Gaza', 'Asia/Gaza'),
+ (265, 'Asia/Harbin', 'Asia/Harbin'),
+ (266, 'Asia/Hebron', 'Asia/Hebron'),
+ (267, 'Asia/Ho_Chi_Minh', 'Asia/Ho_Chi_Minh'),
+ (268, 'Asia/Hong_Kong', 'Asia/Hong_Kong'),
+ (269, 'Asia/Hovd', 'Asia/Hovd'),
+ (270, 'Asia/Irkutsk', 'Asia/Irkutsk'),
+ (271, 'Asia/Istanbul', 'Asia/Istanbul'),
+ (272, 'Asia/Jakarta', 'Asia/Jakarta'),
+ (273, 'Asia/Jayapura', 'Asia/Jayapura'),
+ (274, 'Asia/Jerusalem', 'Asia/Jerusalem'),
+ (275, 'Asia/Kabul', 'Asia/Kabul'),
+ (276, 'Asia/Kamchatka', 'Asia/Kamchatka'),
+ (277, 'Asia/Karachi', 'Asia/Karachi'),
+ (278, 'Asia/Kashgar', 'Asia/Kashgar'),
+ (279, 'Asia/Kathmandu', 'Asia/Kathmandu'),
+ (280, 'Asia/Katmandu', 'Asia/Katmandu'),
+ (281, 'Asia/Khandyga', 'Asia/Khandyga'),
+ (282, 'Asia/Kolkata', 'Asia/Kolkata'),
+ (283, 'Asia/Krasnoyarsk', 'Asia/Krasnoyarsk'),
+ (284, 'Asia/Kuala_Lumpur', 'Asia/Kuala_Lumpur'),
+ (285, 'Asia/Kuching', 'Asia/Kuching'),
+ (286, 'Asia/Kuwait', 'Asia/Kuwait'),
+ (287, 'Asia/Macao', 'Asia/Macao'),
+ (288, 'Asia/Macau', 'Asia/Macau'),
+ (289, 'Asia/Magadan', 'Asia/Magadan'),
+ (290, 'Asia/Makassar', 'Asia/Makassar'),
+ (291, 'Asia/Manila', 'Asia/Manila'),
+ (292, 'Asia/Muscat', 'Asia/Muscat'),
+ (293, 'Asia/Nicosia', 'Asia/Nicosia'),
+ (294, 'Asia/Novokuznetsk', 'Asia/Novokuznetsk'),
+ (295, 'Asia/Novosibirsk', 'Asia/Novosibirsk'),
+ (296, 'Asia/Omsk', 'Asia/Omsk'),
+ (297, 'Asia/Oral', 'Asia/Oral'),
+ (298, 'Asia/Phnom_Penh', 'Asia/Phnom_Penh'),
+ (299, 'Asia/Pontianak', 'Asia/Pontianak'),
+ (300, 'Asia/Pyongyang', 'Asia/Pyongyang'),
+ (301, 'Asia/Qatar', 'Asia/Qatar'),
+ (302, 'Asia/Qostanay', 'Asia/Qostanay'),
+ (303, 'Asia/Qyzylorda', 'Asia/Qyzylorda'),
+ (304, 'Asia/Rangoon', 'Asia/Rangoon'),
+ (305, 'Asia/Riyadh', 'Asia/Riyadh'),
+ (306, 'Asia/Saigon', 'Asia/Saigon'),
+ (307, 'Asia/Sakhalin', 'Asia/Sakhalin'),
+ (308, 'Asia/Samarkand', 'Asia/Samarkand'),
+ (309, 'Asia/Seoul', 'Asia/Seoul'),
+ (310, 'Asia/Shanghai', 'Asia/Shanghai'),
+ (311, 'Asia/Singapore', 'Asia/Singapore'),
+ (312, 'Asia/Srednekolymsk', 'Asia/Srednekolymsk'),
+ (313, 'Asia/Taipei', 'Asia/Taipei'),
+ (314, 'Asia/Tashkent', 'Asia/Tashkent'),
+ (315, 'Asia/Tbilisi', 'Asia/Tbilisi'),
+ (316, 'Asia/Tehran', 'Asia/Tehran'),
+ (317, 'Asia/Tel_Aviv', 'Asia/Tel_Aviv'),
+ (318, 'Asia/Thimbu', 'Asia/Thimbu'),
+ (319, 'Asia/Thimphu', 'Asia/Thimphu'),
+ (320, 'Asia/Tokyo', 'Asia/Tokyo'),
+ (321, 'Asia/Tomsk', 'Asia/Tomsk'),
+ (322, 'Asia/Ujung_Pandang', 'Asia/Ujung_Pandang'),
+ (323, 'Asia/Ulaanbaatar', 'Asia/Ulaanbaatar'),
+ (324, 'Asia/Ulan_Bator', 'Asia/Ulan_Bator'),
+ (325, 'Asia/Urumqi', 'Asia/Urumqi'),
+ (326, 'Asia/Ust-Nera', 'Asia/Ust-Nera'),
+ (327, 'Asia/Vientiane', 'Asia/Vientiane'),
+ (328, 'Asia/Vladivostok', 'Asia/Vladivostok'),
+ (329, 'Asia/Yakutsk', 'Asia/Yakutsk'),
+ (330, 'Asia/Yangon', 'Asia/Yangon'),
+ (331, 'Asia/Yekaterinburg', 'Asia/Yekaterinburg'),
+ (332, 'Asia/Yerevan', 'Asia/Yerevan'),
+ (333, 'Atlantic/Azores', 'Atlantic/Azores'),
+ (334, 'Atlantic/Bermuda', 'Atlantic/Bermuda'),
+ (335, 'Atlantic/Canary', 'Atlantic/Canary'),
+ (336, 'Atlantic/Cape_Verde', 'Atlantic/Cape_Verde'),
+ (337, 'Atlantic/Faeroe', 'Atlantic/Faeroe'),
+ (338, 'Atlantic/Faroe', 'Atlantic/Faroe'),
+ (339, 'Atlantic/Jan_Mayen', 'Atlantic/Jan_Mayen'),
+ (340, 'Atlantic/Madeira', 'Atlantic/Madeira'),
+ (341, 'Atlantic/Reykjavik', 'Atlantic/Reykjavik'),
+ (342, 'Atlantic/South_Georgia', 'Atlantic/South_Georgia'),
+ (343, 'Atlantic/St_Helena', 'Atlantic/St_Helena'),
+ (344, 'Atlantic/Stanley', 'Atlantic/Stanley'),
+ (345, 'Australia/ACT', 'Australia/ACT'),
+ (346, 'Australia/Adelaide', 'Australia/Adelaide'),
+ (347, 'Australia/Brisbane', 'Australia/Brisbane'),
+ (348, 'Australia/Broken_Hill', 'Australia/Broken_Hill'),
+ (349, 'Australia/Canberra', 'Australia/Canberra'),
+ (350, 'Australia/Currie', 'Australia/Currie'),
+ (351, 'Australia/Darwin', 'Australia/Darwin'),
+ (352, 'Australia/Eucla', 'Australia/Eucla'),
+ (353, 'Australia/Hobart', 'Australia/Hobart'),
+ (354, 'Australia/LHI', 'Australia/LHI'),
+ (355, 'Australia/Lindeman', 'Australia/Lindeman'),
+ (356, 'Australia/Lord_Howe', 'Australia/Lord_Howe'),
+ (357, 'Australia/Melbourne', 'Australia/Melbourne'),
+ (358, 'Australia/NSW', 'Australia/NSW'),
+ (359, 'Australia/North', 'Australia/North'),
+ (360, 'Australia/Perth', 'Australia/Perth'),
+ (361, 'Australia/Queensland', 'Australia/Queensland'),
+ (362, 'Australia/South', 'Australia/South'),
+ (363, 'Australia/Sydney', 'Australia/Sydney'),
+ (364, 'Australia/Tasmania', 'Australia/Tasmania'),
+ (365, 'Australia/Victoria', 'Australia/Victoria'),
+ (366, 'Australia/West', 'Australia/West'),
+ (367, 'Australia/Yancowinna', 'Australia/Yancowinna'),
+ (368, 'Brazil/Acre', 'Brazil/Acre'),
+ (369, 'Brazil/DeNoronha', 'Brazil/DeNoronha'),
+ (370, 'Brazil/East', 'Brazil/East'),
+ (371, 'Brazil/West', 'Brazil/West'),
+ (372, 'CET', 'CET'),
+ (373, 'CST6CDT', 'CST6CDT'),
+ (374, 'Canada/Atlantic', 'Canada/Atlantic'),
+ (375, 'Canada/Central', 'Canada/Central'),
+ (376, 'Canada/Eastern', 'Canada/Eastern'),
+ (377, 'Canada/Mountain', 'Canada/Mountain'),
+ (378, 'Canada/Newfoundland', 'Canada/Newfoundland'),
+ (379, 'Canada/Pacific', 'Canada/Pacific'),
+ (380, 'Canada/Saskatchewan', 'Canada/Saskatchewan'),
+ (381, 'Canada/Yukon', 'Canada/Yukon'),
+ (382, 'Chile/Continental', 'Chile/Continental'),
+ (383, 'Chile/EasterIsland', 'Chile/EasterIsland'),
+ (384, 'Cuba', 'Cuba'),
+ (385, 'EET', 'EET'),
+ (386, 'EST', 'EST'),
+ (387, 'EST5EDT', 'EST5EDT'),
+ (388, 'Egypt', 'Egypt'),
+ (389, 'Eire', 'Eire'),
+ (390, 'Etc/GMT', 'Etc/GMT'),
+ (391, 'Etc/GMT+0', 'Etc/GMT+0'),
+ (392, 'Etc/GMT+1', 'Etc/GMT+1'),
+ (393, 'Etc/GMT+10', 'Etc/GMT+10'),
+ (394, 'Etc/GMT+11', 'Etc/GMT+11'),
+ (395, 'Etc/GMT+12', 'Etc/GMT+12'),
+ (396, 'Etc/GMT+2', 'Etc/GMT+2'),
+ (397, 'Etc/GMT+3', 'Etc/GMT+3'),
+ (398, 'Etc/GMT+4', 'Etc/GMT+4'),
+ (399, 'Etc/GMT+5', 'Etc/GMT+5'),
+ (400, 'Etc/GMT+6', 'Etc/GMT+6'),
+ (401, 'Etc/GMT+7', 'Etc/GMT+7'),
+ (402, 'Etc/GMT+8', 'Etc/GMT+8'),
+ (403, 'Etc/GMT+9', 'Etc/GMT+9'),
+ (404, 'Etc/GMT-0', 'Etc/GMT-0'),
+ (405, 'Etc/GMT-1', 'Etc/GMT-1'),
+ (406, 'Etc/GMT-10', 'Etc/GMT-10'),
+ (407, 'Etc/GMT-11', 'Etc/GMT-11'),
+ (408, 'Etc/GMT-12', 'Etc/GMT-12'),
+ (409, 'Etc/GMT-13', 'Etc/GMT-13'),
+ (410, 'Etc/GMT-14', 'Etc/GMT-14'),
+ (411, 'Etc/GMT-2', 'Etc/GMT-2'),
+ (412, 'Etc/GMT-3', 'Etc/GMT-3'),
+ (413, 'Etc/GMT-4', 'Etc/GMT-4'),
+ (414, 'Etc/GMT-5', 'Etc/GMT-5'),
+ (415, 'Etc/GMT-6', 'Etc/GMT-6'),
+ (416, 'Etc/GMT-7', 'Etc/GMT-7'),
+ (417, 'Etc/GMT-8', 'Etc/GMT-8'),
+ (418, 'Etc/GMT-9', 'Etc/GMT-9'),
+ (419, 'Etc/GMT0', 'Etc/GMT0'),
+ (420, 'Etc/Greenwich', 'Etc/Greenwich'),
+ (421, 'Etc/UCT', 'Etc/UCT'),
+ (422, 'Etc/UTC', 'Etc/UTC'),
+ (423, 'Etc/Universal', 'Etc/Universal'),
+ (424, 'Etc/Zulu', 'Etc/Zulu'),
+ (425, 'Europe/Amsterdam', 'Europe/Amsterdam'),
+ (426, 'Europe/Andorra', 'Europe/Andorra'),
+ (427, 'Europe/Astrakhan', 'Europe/Astrakhan'),
+ (428, 'Europe/Athens', 'Europe/Athens'),
+ (429, 'Europe/Belfast', 'Europe/Belfast'),
+ (430, 'Europe/Belgrade', 'Europe/Belgrade'),
+ (431, 'Europe/Berlin', 'Europe/Berlin'),
+ (432, 'Europe/Bratislava', 'Europe/Bratislava'),
+ (433, 'Europe/Brussels', 'Europe/Brussels'),
+ (434, 'Europe/Bucharest', 'Europe/Bucharest'),
+ (435, 'Europe/Budapest', 'Europe/Budapest'),
+ (436, 'Europe/Busingen', 'Europe/Busingen'),
+ (437, 'Europe/Chisinau', 'Europe/Chisinau'),
+ (438, 'Europe/Copenhagen', 'Europe/Copenhagen'),
+ (439, 'Europe/Dublin', 'Europe/Dublin'),
+ (440, 'Europe/Gibraltar', 'Europe/Gibraltar'),
+ (441, 'Europe/Guernsey', 'Europe/Guernsey'),
+ (442, 'Europe/Helsinki', 'Europe/Helsinki'),
+ (443, 'Europe/Isle_of_Man', 'Europe/Isle_of_Man'),
+ (444, 'Europe/Istanbul', 'Europe/Istanbul'),
+ (445, 'Europe/Jersey', 'Europe/Jersey'),
+ (446, 'Europe/Kaliningrad', 'Europe/Kaliningrad'),
+ (447, 'Europe/Kiev', 'Europe/Kiev'),
+ (448, 'Europe/Kirov', 'Europe/Kirov'),
+ (449, 'Europe/Lisbon', 'Europe/Lisbon'),
+ (450, 'Europe/Ljubljana', 'Europe/Ljubljana'),
+ (451, 'Europe/London', 'Europe/London'),
+ (452, 'Europe/Luxembourg', 'Europe/Luxembourg'),
+ (453, 'Europe/Madrid', 'Europe/Madrid'),
+ (454, 'Europe/Malta', 'Europe/Malta'),
+ (455, 'Europe/Mariehamn', 'Europe/Mariehamn'),
+ (456, 'Europe/Minsk', 'Europe/Minsk'),
+ (457, 'Europe/Monaco', 'Europe/Monaco'),
+ (458, 'Europe/Moscow', 'Europe/Moscow'),
+ (459, 'Europe/Nicosia', 'Europe/Nicosia'),
+ (460, 'Europe/Oslo', 'Europe/Oslo'),
+ (461, 'Europe/Paris', 'Europe/Paris'),
+ (462, 'Europe/Podgorica', 'Europe/Podgorica'),
+ (463, 'Europe/Prague', 'Europe/Prague'),
+ (464, 'Europe/Riga', 'Europe/Riga'),
+ (465, 'Europe/Rome', 'Europe/Rome'),
+ (466, 'Europe/Samara', 'Europe/Samara'),
+ (467, 'Europe/San_Marino', 'Europe/San_Marino'),
+ (468, 'Europe/Sarajevo', 'Europe/Sarajevo'),
+ (469, 'Europe/Saratov', 'Europe/Saratov'),
+ (470, 'Europe/Simferopol', 'Europe/Simferopol'),
+ (471, 'Europe/Skopje', 'Europe/Skopje'),
+ (472, 'Europe/Sofia', 'Europe/Sofia'),
+ (473, 'Europe/Stockholm', 'Europe/Stockholm'),
+ (474, 'Europe/Tallinn', 'Europe/Tallinn'),
+ (475, 'Europe/Tirane', 'Europe/Tirane'),
+ (476, 'Europe/Tiraspol', 'Europe/Tiraspol'),
+ (477, 'Europe/Ulyanovsk', 'Europe/Ulyanovsk'),
+ (478, 'Europe/Uzhgorod', 'Europe/Uzhgorod'),
+ (479, 'Europe/Vaduz', 'Europe/Vaduz'),
+ (480, 'Europe/Vatican', 'Europe/Vatican'),
+ (481, 'Europe/Vienna', 'Europe/Vienna'),
+ (482, 'Europe/Vilnius', 'Europe/Vilnius'),
+ (483, 'Europe/Volgograd', 'Europe/Volgograd'),
+ (484, 'Europe/Warsaw', 'Europe/Warsaw'),
+ (485, 'Europe/Zagreb', 'Europe/Zagreb'),
+ (486, 'Europe/Zaporozhye', 'Europe/Zaporozhye'),
+ (487, 'Europe/Zurich', 'Europe/Zurich'),
+ (488, 'GB', 'GB'),
+ (489, 'GB-Eire', 'GB-Eire'),
+ (490, 'GMT', 'GMT'),
+ (491, 'GMT+0', 'GMT+0'),
+ (492, 'GMT-0', 'GMT-0'),
+ (493, 'GMT0', 'GMT0'),
+ (494, 'Greenwich', 'Greenwich'),
+ (495, 'HST', 'HST'),
+ (496, 'Hongkong', 'Hongkong'),
+ (497, 'Iceland', 'Iceland'),
+ (498, 'Indian/Antananarivo', 'Indian/Antananarivo'),
+ (499, 'Indian/Chagos', 'Indian/Chagos'),
+ (500, 'Indian/Christmas', 'Indian/Christmas'),
+ (501, 'Indian/Cocos', 'Indian/Cocos'),
+ (502, 'Indian/Comoro', 'Indian/Comoro'),
+ (503, 'Indian/Kerguelen', 'Indian/Kerguelen'),
+ (504, 'Indian/Mahe', 'Indian/Mahe'),
+ (505, 'Indian/Maldives', 'Indian/Maldives'),
+ (506, 'Indian/Mauritius', 'Indian/Mauritius'),
+ (507, 'Indian/Mayotte', 'Indian/Mayotte'),
+ (508, 'Indian/Reunion', 'Indian/Reunion'),
+ (509, 'Iran', 'Iran'),
+ (510, 'Israel', 'Israel'),
+ (511, 'Jamaica', 'Jamaica'),
+ (512, 'Japan', 'Japan'),
+ (513, 'Kwajalein', 'Kwajalein'),
+ (514, 'Libya', 'Libya'),
+ (515, 'MET', 'MET'),
+ (516, 'MST', 'MST'),
+ (517, 'MST7MDT', 'MST7MDT'),
+ (518, 'Mexico/BajaNorte', 'Mexico/BajaNorte'),
+ (519, 'Mexico/BajaSur', 'Mexico/BajaSur'),
+ (520, 'Mexico/General', 'Mexico/General'),
+ (521, 'NZ', 'NZ'),
+ (522, 'NZ-CHAT', 'NZ-CHAT'),
+ (523, 'Navajo', 'Navajo'),
+ (524, 'PRC', 'PRC'),
+ (525, 'PST8PDT', 'PST8PDT'),
+ (526, 'Pacific/Apia', 'Pacific/Apia'),
+ (527, 'Pacific/Auckland', 'Pacific/Auckland'),
+ (528, 'Pacific/Bougainville', 'Pacific/Bougainville'),
+ (529, 'Pacific/Chatham', 'Pacific/Chatham'),
+ (530, 'Pacific/Chuuk', 'Pacific/Chuuk'),
+ (531, 'Pacific/Easter', 'Pacific/Easter'),
+ (532, 'Pacific/Efate', 'Pacific/Efate'),
+ (533, 'Pacific/Enderbury', 'Pacific/Enderbury'),
+ (534, 'Pacific/Fakaofo', 'Pacific/Fakaofo'),
+ (535, 'Pacific/Fiji', 'Pacific/Fiji'),
+ (536, 'Pacific/Funafuti', 'Pacific/Funafuti'),
+ (537, 'Pacific/Galapagos', 'Pacific/Galapagos'),
+ (538, 'Pacific/Gambier', 'Pacific/Gambier'),
+ (539, 'Pacific/Guadalcanal', 'Pacific/Guadalcanal'),
+ (540, 'Pacific/Guam', 'Pacific/Guam'),
+ (541, 'Pacific/Honolulu', 'Pacific/Honolulu'),
+ (542, 'Pacific/Johnston', 'Pacific/Johnston'),
+ (543, 'Pacific/Kiritimati', 'Pacific/Kiritimati'),
+ (544, 'Pacific/Kosrae', 'Pacific/Kosrae'),
+ (545, 'Pacific/Kwajalein', 'Pacific/Kwajalein'),
+ (546, 'Pacific/Majuro', 'Pacific/Majuro'),
+ (547, 'Pacific/Marquesas', 'Pacific/Marquesas'),
+ (548, 'Pacific/Midway', 'Pacific/Midway'),
+ (549, 'Pacific/Nauru', 'Pacific/Nauru'),
+ (550, 'Pacific/Niue', 'Pacific/Niue'),
+ (551, 'Pacific/Norfolk', 'Pacific/Norfolk'),
+ (552, 'Pacific/Noumea', 'Pacific/Noumea'),
+ (553, 'Pacific/Pago_Pago', 'Pacific/Pago_Pago'),
+ (554, 'Pacific/Palau', 'Pacific/Palau'),
+ (555, 'Pacific/Pitcairn', 'Pacific/Pitcairn'),
+ (556, 'Pacific/Pohnpei', 'Pacific/Pohnpei'),
+ (557, 'Pacific/Ponape', 'Pacific/Ponape'),
+ (558, 'Pacific/Port_Moresby', 'Pacific/Port_Moresby'),
+ (559, 'Pacific/Rarotonga', 'Pacific/Rarotonga'),
+ (560, 'Pacific/Saipan', 'Pacific/Saipan'),
+ (561, 'Pacific/Samoa', 'Pacific/Samoa'),
+ (562, 'Pacific/Tahiti', 'Pacific/Tahiti'),
+ (563, 'Pacific/Tarawa', 'Pacific/Tarawa'),
+ (564, 'Pacific/Tongatapu', 'Pacific/Tongatapu'),
+ (565, 'Pacific/Truk', 'Pacific/Truk'),
+ (566, 'Pacific/Wake', 'Pacific/Wake'),
+ (567, 'Pacific/Wallis', 'Pacific/Wallis'),
+ (568, 'Pacific/Yap', 'Pacific/Yap'),
+ (569, 'Poland', 'Poland'),
+ (570, 'Portugal', 'Portugal'),
+ (571, 'ROC', 'ROC'),
+ (572, 'ROK', 'ROK'),
+ (573, 'Singapore', 'Singapore'),
+ (574, 'Turkey', 'Turkey'),
+ (575, 'UCT', 'UCT'),
+ (576, 'US/Alaska', 'US/Alaska'),
+ (577, 'US/Aleutian', 'US/Aleutian'),
+ (578, 'US/Arizona', 'US/Arizona'),
+ (579, 'US/Central', 'US/Central'),
+ (580, 'US/East-Indiana', 'US/East-Indiana'),
+ (581, 'US/Eastern', 'US/Eastern'),
+ (582, 'US/Hawaii', 'US/Hawaii'),
+ (583, 'US/Indiana-Starke', 'US/Indiana-Starke'),
+ (584, 'US/Michigan', 'US/Michigan'),
+ (585, 'US/Mountain', 'US/Mountain'),
+ (586, 'US/Pacific', 'US/Pacific'),
+ (587, 'US/Samoa', 'US/Samoa'),
+ (588, 'UTC', 'UTC'),
+ (589, 'Universal', 'Universal'),
+ (590, 'W-SU', 'W-SU'),
+ (591, 'WET', 'WET'),
+ (592, 'Zulu', 'Zulu');
+
+
+-- Timeseries: Additional Metadata
+-- ===============================
+
+-- Absorption Cross Section
+
+CREATE TABLE IF NOT EXISTS CS_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT cs_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO CS_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Hearn1961', 'Hearn 1961'),
+ (1, 'CCQM.O3.2019', 'CCQM values of 2019 for O3');
+
+-- Sampling Type (KS: Kind of Sampling)
+
+CREATE TABLE IF NOT EXISTS KS_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT ks_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO KS_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Continuous', 'continuous'),
+ (1, 'Filter', 'filter'),
+ (2, 'Flask', 'flask');
+
+-- Calibration Type
+
+CREATE TABLE IF NOT EXISTS CT_vocabulary (
+ enum_val INT NOT NULL,
+ enum_str character varying(128) NOT NULL,
+ enum_display_str character varying(128) NOT NULL,
+ PRIMARY KEY(enum_val, enum_str),
+ CONSTRAINT ct_enum_val_unique UNIQUE (enum_val)
+);
+
+INSERT INTO CT_vocabulary (enum_val, enum_str, enum_display_str) VALUES
+ (0, 'Automatic', 'automatic'),
+ (1, 'Manual', 'manual');
+
diff --git a/extension/toar_controlled_vocabulary/toar_controlled_vocabulary.control b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary.control
index cd4c997..d6bf2df 100644
--- a/extension/toar_controlled_vocabulary/toar_controlled_vocabulary.control
+++ b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary.control
@@ -1,5 +1,5 @@
# toar_controlled_vocabulary extension
comment = 'TOAR controlled vocabulary'
-default_version = '0.7.6'
-module_pathname = '$libdir/toar_controlled_vocabulary-0.7.6'
+default_version = '0.7.7'
+module_pathname = '$libdir/toar_controlled_vocabulary-0.7.7'
relocatable = false
--
GitLab
From 40e491889ff66293c34d0b63d652174ed15834fe Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Fri, 3 Jan 2025 16:33:44 +0000
Subject: [PATCH 19/36] added some local corrections to the controlled
vocabulary that had been lost
---
.../toar_controlled_vocabulary--0.7.6--0.7.7.sql | 6 ++++++
.../toar_controlled_vocabulary--0.7.7.sql | 4 ++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql
index 258d6d9..5faa841 100644
--- a/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql
+++ b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.6--0.7.7.sql
@@ -20,6 +20,12 @@ SET SCHEMA 'toar_convoc';
-- Stationmeta
-- ===========
+-- climatic zones
+-- see: IPCC Climate Zone Map reported in Figure 3A.5.1 in Chapter 3 of Vol.4 of 2019 Refinement
+
+UPDATE CZ_vocabulary SET enum_display_str='-1 (undefined)' WHERE enum_val=-1;
+UPDATE CZ_vocabulary SET enum_display_str='0 (unclassified)' WHERE enum_val=0;
+
-- Station HTAP Regions (TIER1)
-- The integer denoting the “tier1” region defined in the task force on hemispheric transport of air pollution (TFHTAP) coordinated model studies.
diff --git a/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql
index 269164a..0acde35 100644
--- a/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql
+++ b/extension/toar_controlled_vocabulary/toar_controlled_vocabulary--0.7.7.sql
@@ -205,8 +205,8 @@ CREATE TABLE IF NOT EXISTS CZ_vocabulary (
);
INSERT INTO CZ_vocabulary (enum_val, enum_str, enum_display_str) VALUES
- (-1, 'Undefined', 'undefined'),
- ( 0, 'Unclassified', 'unclassified'),
+ (-1, 'Undefined', '-1 (undefined)'),
+ ( 0, 'Unclassified', '0 (unclassified)'),
( 1, 'TropicalMontane', '1 (tropical montane)'),
( 2, 'TropicalWet', '2 (tropical wet)'),
( 3, 'TropicalMoist', '3 (tropical moist)'),
--
GitLab
From a3b6af7dfca5e96b279012f0ad0f74a8050ba998 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Wed, 29 Jan 2025 12:37:28 +0000
Subject: [PATCH 20/36] #154: list of contributors can be added for a request
id from the analysis service
---
tests/fixtures/timeseries/timeseries.json | 6 +-
tests/fixtures/toardb_pytest.psql | 24 ++++++-
tests/test_timeseries.py | 83 ++++++++++++++++++++++-
toardb/timeseries/crud.py | 37 ++++++++--
toardb/timeseries/models.py | 1 +
toardb/timeseries/timeseries.py | 19 +++++-
toardb/utils/utils.py | 16 +++--
7 files changed, 172 insertions(+), 14 deletions(-)
diff --git a/tests/fixtures/timeseries/timeseries.json b/tests/fixtures/timeseries/timeseries.json
index fa3d164..bd7c59b 100644
--- a/tests/fixtures/timeseries/timeseries.json
+++ b/tests/fixtures/timeseries/timeseries.json
@@ -11,7 +11,8 @@
"data_origin": 0,
"data_origin_type": 0,
"sampling_height": 7,
- "additional_metadata": {}
+ "additional_metadata": {},
+ "programme_id": 0
},
{
"station_id": 3,
@@ -32,6 +33,7 @@
"Data level": "2",
"Frameworks": "GAW-WDCRG NOAA-ESRL",
"Station code": "XXX",
- "Station name": "Secret" } }
+ "Station name": "Secret" } },
+ "programme_id": 0
}
]
diff --git a/tests/fixtures/toardb_pytest.psql b/tests/fixtures/toardb_pytest.psql
index 372960d..e7da295 100644
--- a/tests/fixtures/toardb_pytest.psql
+++ b/tests/fixtures/toardb_pytest.psql
@@ -25,6 +25,16 @@ CREATE SCHEMA IF NOT EXISTS toar_convoc;
ALTER SCHEMA toar_convoc OWNER TO postgres;
+--
+-- Name: services; Type: SCHEMA; Schema: -; Owner: postgres
+--
+
+CREATE SCHEMA IF NOT EXISTS services;
+
+
+ALTER SCHEMA services OWNER TO postgres;
+
+
--
-- Name: address_standardizer; Type: EXTENSION; Schema: -; Owner: -
--
@@ -102,6 +112,18 @@ CREATE EXTENSION IF NOT EXISTS postgis_topology WITH SCHEMA topology;
COMMENT ON EXTENSION postgis_topology IS 'PostGIS topology spatial types and functions';
+--
+-- List of contributors for request of special services
+-- here: s1: analysis-service
+--
+
+CREATE TABLE IF NOT EXISTS services.s1_contributors (
+ request_id character varying(36) NOT NULL,
+ timeseries_ids bigint[],
+ PRIMARY KEY(request_id)
+);
+
+
--
-- Name: toar_controlled_vocabulary; Type: EXTENSION -- faked; Schema: -; Owner: -
--
@@ -4064,5 +4086,5 @@ ALTER TABLE ONLY public.timeseries
ADD CONSTRAINT timeseries_variable_id_dd9603f5_fk_variables_id FOREIGN KEY (variable_id) REFERENCES public.variables(id) DEFERRABLE INITIALLY DEFERRED;
ALTER DATABASE postgres
- SET search_path=public,tiger,toar_convoc;
+ SET search_path=public,tiger,toar_convoc,services;
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index 7270131..b27d5fc 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -4,7 +4,8 @@
import pytest
import json
from sqlalchemy import insert
-from toardb.timeseries.models import Timeseries, timeseries_timeseries_roles_table
+from toardb.timeseries.models import Timeseries, timeseries_timeseries_roles_table, \
+ s1_contributors_table
from toardb.timeseries.models_programme import TimeseriesProgramme
from toardb.timeseries.models_role import TimeseriesRole
from toardb.stationmeta.models import StationmetaCore, StationmetaGlobal
@@ -165,6 +166,12 @@ class TestApps:
for entry in metajson:
db.execute(insert(timeseries_timeseries_roles_table).values(timeseries_id=entry["timeseries_id"], role_id=entry["role_id"]))
db.execute("COMMIT")
+ infilename = "tests/fixtures/timeseries/timeseries_contributors.json"
+ with open(infilename) as f:
+ metajson=json.load(f)
+ for entry in metajson:
+ db.execute(insert(s1_contributors_table).values(request_id=entry["request_id"], timeseries_ids=entry["timeseries_ids"]))
+ db.execute("COMMIT")
def test_get_timeseries(self, client, db):
@@ -1011,8 +1018,82 @@ class TestApps:
expected_status_code = 400
assert response.status_code == expected_status_code
expected_response = 'not a valid format: CMOR'
+ assert response.json() == expected_response
+
+
+ def test_register_contributors_list(self, client, db):
+ response = client.post("/timeseries/register_timeseries_list_of_contributors/5f0df73a-bd0f-48b9-bb17-d5cd36f89598",
+ data='''[1,2]''',
+ headers={"email": "s.schroeder@fz-juelich.de"} )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = '5f0df73a-bd0f-48b9-bb17-d5cd36f89598 successfully registered.'
assert response.json() == expected_response
+
+ def test_register_duplicate_contributors_list(self, client, db):
+ response = client.post("/timeseries/register_timeseries_list_of_contributors/7f0df73a-bd0f-48b9-bb17-d5cd36f89598",
+ data='''[1,2]''',
+ headers={"email": "s.schroeder@fz-juelich.de"} )
+ expected_status_code = 443
+ assert response.status_code == expected_status_code
+ expected_response = '7f0df73a-bd0f-48b9-bb17-d5cd36f89598 already registered.'
+ assert response.json() == expected_response
+
+
+ def test_request_registered_contributors_list_json(self, client, db):
+ response = client.get("/timeseries/request_timeseries_list_of_contributors/7f0df73a-bd0f-48b9-bb17-d5cd36f89598?format=json")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = [
+ {'contact': {'id': 5,
+ 'organisation': {'city': 'Jülich',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'id': 2,
+ 'kind': 'research',
+ 'longname': 'Forschungszentrum Jülich',
+ 'name': 'FZJ',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße'
+ },
+ 'person': None
+ },
+ 'id': 1,
+ 'role': 'resource provider',
+ 'status': 'active'
+ },
+ {'contact': {'id': 4,
+ 'organisation': {'city': 'Dessau-Roßlau',
+ 'contact_url': 'mailto:immission@uba.de',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'id': 1,
+ 'kind': 'government',
+ 'longname': 'Umweltbundesamt',
+ 'name': 'UBA',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1'
+ },
+ 'person': None
+ },
+ 'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active'
+ }
+ ]
+ assert response.json() == expected_response
+
+
+ def test_request_registered_contributors_list_text(self, client, db):
+ response = client.get("/timeseries/request_timeseries_list_of_contributors/7f0df73a-bd0f-48b9-bb17-d5cd36f89598?format=text")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = 'organisations: Forschungszentrum Jülich;Umweltbundesamt'
+ assert response.json() == expected_response
+
+
# 3. tests updating timeseries metadata
def test_patch_timeseries_no_description(self, client, db):
diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py
index e8cda0b..a2e7332 100644
--- a/toardb/timeseries/crud.py
+++ b/toardb/timeseries/crud.py
@@ -9,6 +9,7 @@ Create, Read, Update, Delete functionality
from sqlalchemy import insert, select, and_, func, text
from sqlalchemy.orm import Session, load_only
from sqlalchemy.util import _collections
+from sqlalchemy.exc import IntegrityError
from geoalchemy2.elements import WKBElement, WKTElement
from fastapi import File, UploadFile
from fastapi.responses import JSONResponse
@@ -16,7 +17,7 @@ import datetime as dt
import json
from . import models
from .models import TimeseriesChangelog, timeseries_timeseries_roles_table, \
- timeseries_timeseries_annotations_table
+ timeseries_timeseries_annotations_table, s1_contributors_table
from toardb.stationmeta.models import StationmetaCore, StationmetaGlobal
from toardb.stationmeta.schemas import get_coordinates_from_geom, get_geom_from_coordinates, get_coordinates_from_string
from toardb.stationmeta.crud import get_stationmeta_by_id, get_stationmeta_core, station_id_exists, get_stationmeta_changelog
@@ -586,9 +587,7 @@ def get_contributors_string(programmes, roles):
return result
-def get_request_contributors(db: Session, format: str = 'text', input_handle: UploadFile = File(...)):
- f = input_handle.file
- timeseries_ids = [int(line.strip()) for line in f.readlines()]
+def get_contributors_list(db: Session, timeseries_ids, format: str = 'text'):
# get time series' programmes
# join(models.Timeseries, Timeseries.programme_id == TimeseriesProgramme.id) is implicit given due to the foreign key
programmes = db.query(models.TimeseriesProgramme) \
@@ -612,6 +611,36 @@ def get_request_contributors(db: Session, format: str = 'text', input_handle: Up
return result
+def get_request_contributors(db: Session, format: str = 'text', input_handle: UploadFile = File(...)):
+ f = input_handle.file
+ timeseries_ids = [int(line.strip()) for line in f.readlines()]
+ return get_contributors_list(db, timeseries_ids, format)
+
+
+def get_registered_request_contributors(db: Session, rid, format: str = 'text'):
+ timeseries_ids = db.execute(select([s1_contributors_table]).\
+ where(s1_contributors_table.c.request_id == rid)).mappings().first()['timeseries_ids']
+ return get_contributors_list(db, timeseries_ids, format)
+
+
+def register_request_contributors(db: Session, rid, ids):
+ try:
+ db.execute(insert(s1_contributors_table).values(request_id=rid, timeseries_ids=ids))
+ db.commit()
+ status_code = 200
+ message=f'{rid} successfully registered.'
+ except IntegrityError as e:
+ error_code = e.orig.pgcode
+ if error_code == "23505":
+ status_code = 443
+ message=f'{rid} already registered.'
+ else:
+ status_code = 442
+ message=f'database error: error_code'
+ result = JSONResponse(status_code=status_code, content=message)
+ return result
+
+
# is this internal, or should this also go to public REST api?
def get_timeseries_roles(db: Session, timeseries_id: int):
return db.execute(select([timeseries_timeseries_roles_table]).where(timeseries_timeseries_roles_table.c.timeseries_id == timeseries_id))
diff --git a/toardb/timeseries/models.py b/toardb/timeseries/models.py
index f697fb3..121bcc7 100644
--- a/toardb/timeseries/models.py
+++ b/toardb/timeseries/models.py
@@ -6,6 +6,7 @@ from .models_role import TimeseriesRole, timeseries_timeseries_roles_table
from .models_annotation import TimeseriesAnnotation, timeseries_timeseries_annotations_table
from .models_programme import TimeseriesProgramme
from .models_changelog import TimeseriesChangelog
+from .models_contributor import s1_contributors_table
from toardb.base import Base
from sqlalchemy import Table, Column, Integer, String
diff --git a/toardb/timeseries/timeseries.py b/toardb/timeseries/timeseries.py
index d5b5e21..1a40821 100644
--- a/toardb/timeseries/timeseries.py
+++ b/toardb/timeseries/timeseries.py
@@ -24,7 +24,8 @@ from toardb.data.models import Data
from toardb.utils.utils import (
get_str_from_value,
get_admin_access_rights,
- get_timeseries_md_change_access_rights
+ get_timeseries_md_change_access_rights,
+ get_register_contributors_access_rights
)
@@ -93,17 +94,33 @@ def get_timeseries(station_id: int, variable_id: int, resource_provider: str = N
raise HTTPException(status_code=404, detail="Timeseries not found.")
return db_timeseries
+
@router.get('/timeseries_changelog/{timeseries_id}', response_model=List[schemas.TimeseriesChangelog])
def get_timeseries_changelog(timeseries_id: int, db: Session = Depends(get_db)):
db_changelog = crud.get_timeseries_changelog(db, timeseries_id=timeseries_id)
return db_changelog
+
@router.get('/timeseries_programme/{name}', response_model=List[schemas.TimeseriesProgramme])
def get_timeseries_programme(name: str, db: Session = Depends(get_db)):
db_programme = crud.get_timeseries_programme(db, name=name)
return db_programme
+@router.get('/timeseries/request_timeseries_list_of_contributors/{rid}', response_model=Union[str,List[schemas.Contributors]])
+def get_registered_request_contributors(rid: str, format: str = 'text', db: Session = Depends(get_db)):
+ contributors = crud.get_registered_request_contributors(db, rid=rid, format=format)
+ return contributors
+
+
+@router.post('/timeseries/register_timeseries_list_of_contributors/{rid}')
+def register_request_contributors(rid: str,
+ ids: List[int],
+ access: dict = Depends(get_register_contributors_access_rights),
+ db: Session = Depends(get_db)):
+ return crud.register_request_contributors(db, rid=rid, ids=ids)
+
+
@router.post('/timeseries/request_contributors', response_model=Union[str,List[schemas.Contributors]])
def get_request_contributors(format: str = 'text', file: UploadFile = File(...), db: Session = Depends(get_db)):
contributors = crud.get_request_contributors(db, format=format, input_handle=file)
diff --git a/toardb/utils/utils.py b/toardb/utils/utils.py
index 5bdc457..e84a4b2 100644
--- a/toardb/utils/utils.py
+++ b/toardb/utils/utils.py
@@ -16,6 +16,8 @@ import requests
import datetime as dt
from toardb.utils.settings import base_geopeas_url, userinfo_endpoint
+
+# the following statement only if not in testing (pytest) mode!
from toardb.utils.database import get_db
from toardb.contacts.models import Contact, Organisation, Person
from toardb.timeseries.models import Timeseries, TimeseriesRole, timeseries_timeseries_roles_table
@@ -38,7 +40,7 @@ def get_access_rights(request: Request, access_right: str = 'admin'):
if status_code != 401:
userinfo = userinfo.json()
if "eduperson_entitlement" in userinfo and \
- f"urn:geant:helmholtz.de:res:toar-data:{access_right}#login-dev.helmholtz.de" \
+ f"urn:geant:helmholtz.de:res:toar-data{access_right}#login.helmholtz.de" \
in userinfo["eduperson_entitlement"]:
user_name = userinfo["name"]
user_email = userinfo["email"]
@@ -59,20 +61,24 @@ def get_access_rights(request: Request, access_right: str = 'admin'):
def get_admin_access_rights(request: Request):
- return get_access_rights(request, 'admin')
+ return get_access_rights(request, ':admin')
def get_station_md_change_access_rights(request: Request):
- return get_access_rights(request, 'station-md-change')
+ return get_access_rights(request, ':station-md-change')
def get_timeseries_md_change_access_rights(request: Request):
- return get_access_rights(request, 'timeseries-md-change')
+ return get_access_rights(request, ':timeseries-md-change')
def get_data_change_access_rights(request: Request):
- return get_access_rights(request, 'data-change')
+ return get_access_rights(request, ':data-change')
+
+
+def get_register_contributors_access_rights(request: Request):
+ return get_access_rights(request, ':contributors-register')
# function to return code for given value
--
GitLab
From 49c08234db471c3cd51f0f4f543f0441cea88f69 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Wed, 29 Jan 2025 12:45:18 +0000
Subject: [PATCH 21/36] #154: the last commit was lacking two files
---
.../timeseries/timeseries_contributors.json | 6 ++++++
toardb/timeseries/models_contributor.py | 16 ++++++++++++++++
2 files changed, 22 insertions(+)
create mode 100644 tests/fixtures/timeseries/timeseries_contributors.json
create mode 100644 toardb/timeseries/models_contributor.py
diff --git a/tests/fixtures/timeseries/timeseries_contributors.json b/tests/fixtures/timeseries/timeseries_contributors.json
new file mode 100644
index 0000000..d010d10
--- /dev/null
+++ b/tests/fixtures/timeseries/timeseries_contributors.json
@@ -0,0 +1,6 @@
+[
+ {
+ "request_id": "7f0df73a-bd0f-48b9-bb17-d5cd36f89598",
+ "timeseries_ids": [1, 2]
+ }
+]
diff --git a/toardb/timeseries/models_contributor.py b/toardb/timeseries/models_contributor.py
new file mode 100644
index 0000000..228f259
--- /dev/null
+++ b/toardb/timeseries/models_contributor.py
@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2021 Forschungszentrum Jülich GmbH
+# SPDX-License-Identifier: MIT
+
+"""
+class TimeseriesContributorList (Base)
+======================================
+"""
+from sqlalchemy import Column, Integer, String, ARRAY, Table
+from toardb.base import Base
+
+
+s1_contributors_table = Table('services.s1_contributors', Base.metadata,
+ Column('request_id', String(32), primary_key=True, nullable=False),
+ Column('timeseries_ids',ARRAY(Integer))
+)
+
--
GitLab
From 4748c15d1c9d0dc0c8e368cce563f4ae3a0e22dd Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Wed, 29 Jan 2025 12:56:07 +0000
Subject: [PATCH 22/36] #154: Increase the length of the database character
string used to store the request ID
---
toardb/timeseries/models_contributor.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/toardb/timeseries/models_contributor.py b/toardb/timeseries/models_contributor.py
index 228f259..342fb83 100644
--- a/toardb/timeseries/models_contributor.py
+++ b/toardb/timeseries/models_contributor.py
@@ -10,7 +10,7 @@ from toardb.base import Base
s1_contributors_table = Table('services.s1_contributors', Base.metadata,
- Column('request_id', String(32), primary_key=True, nullable=False),
+ Column('request_id', String(36), primary_key=True, nullable=False),
Column('timeseries_ids',ARRAY(Integer))
)
--
GitLab
From bee973e9a96037413a22544f3732680f90b3ac27 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Wed, 29 Jan 2025 13:15:58 +0000
Subject: [PATCH 23/36] pick working CI settings and files from master
---
.gitlab-ci.yml | 76 +++++++++++++++++++++++++++++++-
CI/do_pytest_coverage.sh | 45 +++++++++++++++++++
CI/update_badge.sh | 93 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 212 insertions(+), 2 deletions(-)
create mode 100644 CI/do_pytest_coverage.sh
create mode 100644 CI/update_badge.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 460f754..bae2cc3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,6 +8,7 @@ variables:
PGPASSWORD: postgres
stages:
+ - init
- test
- deploy
- pages
@@ -17,7 +18,26 @@ cache:
paths:
- public/docs/
-#### Test ####
+
+### Static Badges ###
+version:
+ stage: init
+ tags:
+ - linux
+ - opensuse
+ only:
+ - master
+ - tags
+ script:
+ - chmod +x ./CI/update_badge.sh
+ artifacts:
+ name: pages
+ when: always
+ paths:
+ - badges/
+
+
+#### Tests ####
test:
stage: test
services:
@@ -27,6 +47,9 @@ test:
- public-docker
variables:
FAILURE_THRESHOLD: 85
+ before_script:
+ - chmod +x ./CI/update_badge.sh
+ - ./CI/update_badge.sh > /dev/null
script:
- apt-get update && apt-get install -y postgresql-client
- pip install --upgrade pip
@@ -34,6 +57,43 @@ test:
- psql -h postgres -U $POSTGRES_USER -d $POSTGRES_DB -f tests/fixtures/toardb_pytest.psql
- chmod +x ./CI/do_pytest.sh
- ./CI/do_pytest.sh
+ after_script:
+ - ./CI/update_badge.sh > /dev/null
+ artifacts:
+ name: pages
+ when: always
+ paths:
+ - badges/
+
+coverage:
+ stage: test
+ services:
+ - name: postgis/postgis
+ alias: postgres
+ tags:
+ - public-docker
+ variables:
+ FAILURE_THRESHOLD: 50
+ COVERAGE_PASS_THRESHOLD: 45
+ before_script:
+ - chmod +x ./CI/update_badge.sh
+ - ./CI/update_badge.sh > /dev/null
+ script:
+ - apt-get update && apt-get install -y postgresql-client
+ - pip install --upgrade pip
+ - pip install coverage
+ - pip install --no-cache-dir -r requirements.txt
+ - psql -h postgres -U $POSTGRES_USER -d $POSTGRES_DB -f tests/fixtures/toardb_pytest.psql
+ - chmod +x ./CI/do_pytest_coverage.sh
+ - ./CI/do_pytest_coverage.sh
+ after_script:
+ - ./CI/update_badge.sh > /dev/null
+ artifacts:
+ name: pages
+ when: always
+ paths:
+ - badges/
+ - coverage/
#### Documentation ####
docs:
@@ -68,6 +128,12 @@ pages:
- dev
when: always
script:
+ - mkdir -p public/badges/
+ - cp -af badges/badge_*.svg public/badges/
+ - ls public/badges/
+ - mkdir -p public/coverage
+ - cp -af coverage/. public/coverage
+ - ls public/coverage
- mkdir -p public/docs
- cp -af docs/_build/html/* public/docs
- ls public/docs/
@@ -76,4 +142,10 @@ pages:
when: always
paths:
- public/
-
+ - badges/
+ - coverage/
+ cache:
+ key: old-pages
+ paths:
+ - public/badges/
+ - public/coverage/
diff --git a/CI/do_pytest_coverage.sh b/CI/do_pytest_coverage.sh
new file mode 100644
index 0000000..537abad
--- /dev/null
+++ b/CI/do_pytest_coverage.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+export PYTHONPATH=/builds/esde/toar-data/toardb_fastapi/
+
+# run coverage twice, 1) for html deploy 2) for success evaluation
+coverage run -m pytest tests
+coverage html
+coverage report | tee coverage_results.out
+
+IS_FAILED=$?
+
+# move html coverage report
+mkdir coverage/
+BRANCH_NAME=$( echo -e "${CI_COMMIT_REF_NAME////_}")
+mkdir coverage/${BRANCH_NAME}
+mkdir coverage/recent
+cp -r htmlcov/* coverage/${BRANCH_NAME}/.
+cp -r htmlcov/* coverage/recent/.
+if [[ "${CI_COMMIT_REF_NAME}" = "master" ]]; then
+ cp -r htmlcov/* coverage/.
+fi
+
+# extract coverage information
+COVERAGE_RATIO="$(grep -oP '\d+\%' coverage_results.out | tail -1)"
+COVERAGE_RATIO="$(echo ${COVERAGE_RATIO} | (grep -oP '\d*'))"
+
+# report
+if [[ ${IS_FAILED} == 0 ]]; then
+ if [[ ${COVERAGE_RATIO} -lt ${COVERAGE_PASS_THRESHOLD} ]]; then
+ echo "only ${COVERAGE_RATIO}% covered"
+ echo "incomplete" > status.txt
+ echo "${COVERAGE_RATIO}%25" > incomplete.txt
+ if [[ ${COVERAGE_RATIO} -lt ${FAILURE_THRESHOLD} ]]; then
+ echo -e "\033[1;31monly ${COVERAGE_RATIO}% covered!!\033[0m"
+ exit 1
+ fi
+ else
+ echo "passed"
+ echo "success" > status.txt
+ echo "${COVERAGE_RATIO}%25" > success.txt
+ fi
+ exit 0
+else
+ echo "not passed"
+ exit 1
+fi
diff --git a/CI/update_badge.sh b/CI/update_badge.sh
new file mode 100644
index 0000000..c8b1101
--- /dev/null
+++ b/CI/update_badge.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+# 'running', 'success' or 'failure' is in this file
+if [[ -e status.txt ]]; then
+ EXIT_STATUS=`cat status.txt`
+else
+ EXIT_STATUS="running"
+fi
+
+printf "%s\n" ${EXIT_STATUS}
+
+# fetch badge_status
+BADGE_STATUS="${CI_COMMIT_REF_NAME}:${CI_JOB_NAME}"
+# replace - with --
+BADGE_STATUS=$( echo -e "${BADGE_STATUS//\-/--}")
+
+
+# Set values for shields.io fields based on STATUS
+if [[ ${EXIT_STATUS} = "running" ]]; then
+ BADGE_SUBJECT="running"
+ BADGE_COLOR="lightgrey"
+elif [[ ${EXIT_STATUS} = "failure" ]]; then
+ BADGE_SUBJECT="failed"
+ BADGE_COLOR="red"
+elif [[ ${EXIT_STATUS} = "success" ]]; then
+ BADGE_SUBJECT="passed"
+ BADGE_COLOR="brightgreen"
+ if [[ -e success.txt ]]; then
+ SUCCESS_MESSAGE=`cat success.txt`
+ BADGE_SUBJECT="${SUCCESS_MESSAGE}"
+ fi
+elif [[ ${EXIT_STATUS} = "incomplete" ]]; then
+ EXIT_STATUS_MESSAGE=`cat incomplete.txt`
+ BADGE_SUBJECT="${EXIT_STATUS_MESSAGE}"
+ EXIT_STATUS_RATIO="$(echo ${EXIT_STATUS_MESSAGE} | (grep -oP '\d*') | head -1)"
+ printf "%s\n" ${EXIT_STATUS_RATIO}
+ if [[ "${EXIT_STATUS_RATIO}" -lt "${FAILURE_THRESHOLD}" ]]; then
+ BADGE_COLOR="red"
+ else
+ BADGE_COLOR="yellow"
+ fi
+else
+ exit 1
+fi
+
+# load additional options
+while getopts b:c:s: option
+do
+ case ${option} in
+ b) BADGE_STATUS=$( echo -e "${OPTARG//\-/--}");;
+ c) BADGE_COLOR=$( echo -e "${OPTARG//\-/--}");;
+ s) BADGE_SUBJECT=$( echo -e "${OPTARG//\-/--}");;
+ esac
+done
+
+
+# Set filename for the badge (i.e. 'ci-test-branch-job.svg')
+CI_COMMIT_REF_NAME_NO_SLASH="$( echo -e "${CI_COMMIT_REF_NAME}" | tr '/' '_' )"
+if [[ ${BADGE_STATUS} = "version" ]]; then
+ BADGE_FILENAME="badge_version.svg"
+else
+ BADGE_FILENAME="badge_${CI_COMMIT_REF_NAME_NO_SLASH}-${CI_JOB_NAME}.svg"
+fi
+RECENT_BADGE_FILENAME="badge_recent-${CI_JOB_NAME}.svg"
+
+# Get the badge from shields.io
+SHIELDS_IO_NAME=${BADGE_STATUS}-${BADGE_SUBJECT}-${BADGE_COLOR}.svg
+printf "%s\n" "INFO: Fetching badge ${SHIELDS_IO_NAME} from shields.io to ${BADGE_FILENAME}."
+printf "%s\n" "${SHIELDS_IO_NAME//\_/__}"
+printf "%s\n" "${SHIELDS_IO_NAME//\#/%23}"
+
+SHIELDS_IO_NAME="$( echo -e "${SHIELDS_IO_NAME//\_/__}" )"
+SHIELDS_IO_NAME="$( echo -e "${SHIELDS_IO_NAME//\#/%23}")"
+curl "https://img.shields.io/badge/${SHIELDS_IO_NAME}" > ${BADGE_FILENAME}
+echo "https://img.shields.io/badge/${SHIELDS_IO_NAME}"
+SHIELDS_IO_NAME_RECENT="RECENT:${SHIELDS_IO_NAME}"
+curl "https://img.shields.io/badge/${SHIELDS_IO_NAME_RECENT}" > ${RECENT_BADGE_FILENAME}
+echo "${SHIELDS_IO_NAME_RECENT}" > testRecentName.txt
+
+#
+if [[ ! -d ./badges ]]; then
+ # Control will enter here if $DIRECTORY doesn't exist.
+ mkdir badges/
+fi
+mv ${BADGE_FILENAME} ./badges/.
+
+# replace outdated recent badge by new badge
+mv ${RECENT_BADGE_FILENAME} ./badges/${RECENT_BADGE_FILENAME}
+
+# set status to failed, this will be overwritten if job ended with exitcode 0
+echo "failed" > status.txt
+
+exit 0
--
GitLab
From 7155f6d019fc2d1bfa7a9568559c108beee7c487 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Mon, 3 Feb 2025 13:01:27 +0000
Subject: [PATCH 24/36] force application not to use a non-existing temporary
contributors table
---
production_tests.sh | 2 ++
toardb/timeseries/models_contributor.py | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/production_tests.sh b/production_tests.sh
index f4a4364..0222252 100755
--- a/production_tests.sh
+++ b/production_tests.sh
@@ -50,3 +50,5 @@ curl -X POST -H 'Content-Type: multipart/form-data; charset=utf-8; boundary=__X_
curl -X PATCH -H 'Content-Type: multipart/form-data; charset=utf-8; boundary=__X_PAW_BOUNDARY__' -F "file=@o3_CO002_2012_2017_v1-0.dat" "http://127.0.0.1:8000/data/timeseries/?description=fixed%20formatting%20errors%20on%20data&version=000001.000001.00000000000000"
# add one record (including changelog entry)
curl -X POST -H "Content-Type:application/json" "http://127.0.0.1:8000/data/timeseries/record/?series_id=96&datetime=2021-08-23%2015:00:00&value=67.3&flag=OK&version=000001.000001.00000000000000"
+# register list of timeseries (ids) contributing to a service request
+curl -X POST -H "Content-Type:application/json" -d '''[1,2,3]''' "http://127.0.0.1:8000/timeseries/register_timeseries_list_of_contributors/5f0df73a-bd0f-48b9-bb17-d5cd36f89598"
diff --git a/toardb/timeseries/models_contributor.py b/toardb/timeseries/models_contributor.py
index 342fb83..de4ccb5 100644
--- a/toardb/timeseries/models_contributor.py
+++ b/toardb/timeseries/models_contributor.py
@@ -9,7 +9,7 @@ from sqlalchemy import Column, Integer, String, ARRAY, Table
from toardb.base import Base
-s1_contributors_table = Table('services.s1_contributors', Base.metadata,
+s1_contributors_table = Table('s1_contributors', Base.metadata,
Column('request_id', String(36), primary_key=True, nullable=False),
Column('timeseries_ids',ARRAY(Integer))
)
--
GitLab
From 01dd35baa3ad5a1b8b2cdbe08e714d232e097820 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Mon, 3 Feb 2025 15:16:59 +0000
Subject: [PATCH 25/36] added fixtures to reach a 100 % coverage for
data/schemas.py
---
tests/fixtures/data/data.json | 42 +++++++++++++++++++++++++++++++++++
tests/test_data.py | 8 ++++++-
2 files changed, 49 insertions(+), 1 deletion(-)
diff --git a/tests/fixtures/data/data.json b/tests/fixtures/data/data.json
index 5b28000..db41d6d 100644
--- a/tests/fixtures/data/data.json
+++ b/tests/fixtures/data/data.json
@@ -124,5 +124,47 @@
"flags":0,
"timeseries_id":2,
"version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2014-12-16 23:00:00+00",
+ "value":13.734,
+ "flags":1,
+ "timeseries_id":2,
+ "version":"000000.000000.20141217235502"
+ },
+ {
+ "datetime":"2014-12-17 00:00:00+00",
+ "value":7.848,
+ "flags":2,
+ "timeseries_id":2,
+ "version":"000000.000000.20141217235502"
+ },
+ {
+ "datetime":"2014-12-17 01:00:00+00",
+ "value":15.696,
+ "flags":2,
+ "timeseries_id":2,
+ "version":"000000.000000.20141217235502"
+ },
+ {
+ "datetime":"2014-12-17 02:00:00+00",
+ "value":11.772,
+ "flags":0,
+ "timeseries_id":2,
+ "version":"000000.000000.20141217235502"
+ },
+ {
+ "datetime":"2014-12-17 03:00:00+00",
+ "value":13.734,
+ "flags":1,
+ "timeseries_id":2,
+ "version":"000000.000000.20141217235502"
+ },
+ {
+ "datetime":"2014-12-17 04:00:00+00",
+ "value":19.62,
+ "flags":0,
+ "timeseries_id":2,
+ "version":"000000.000000.20141217235502"
}
]
diff --git a/tests/test_data.py b/tests/test_data.py
index 2e83788..c86186e 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -942,7 +942,13 @@ class TestApps:
{'datetime': '2013-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'questionable validated flagged', 'timeseries_id': 2, 'version': '1.0'},
{'datetime': '2013-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'questionable validated unconfirmed', 'timeseries_id': 2, 'version': '1.0'},
{'datetime': '2013-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'questionable validated flagged', 'timeseries_id': 2, 'version': '1.0'},
- {'datetime': '2013-12-17T06:00:00+00:00', 'value': 5.886, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'}]
+ {'datetime': '2013-12-17T06:00:00+00:00', 'value': 5.886, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2014-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '0.0.20141217235502'},
+ {'datetime': '2014-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated modified', 'timeseries_id': 2, 'version': '0.0.20141217235502'},
+ {'datetime': '2014-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated modified', 'timeseries_id': 2, 'version': '0.0.20141217235502'},
+ {'datetime': '2014-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '0.0.20141217235502'},
+ {'datetime': '2014-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '0.0.20141217235502'},
+ {'datetime': '2014-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '0.0.20141217235502'}]
}
assert response.json() == expected_response
--
GitLab
From d273c55acf18eaba11087f2cbd83cc1ae2bb26ee Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Mon, 10 Feb 2025 16:55:39 +0000
Subject: [PATCH 26/36] started fixing the patching of a station's annotations
---
tests/test_stationmeta.py | 27 +++++++++++++++++++++++++++
toardb/stationmeta/schemas.py | 23 ++++++++++++++++++++---
toardb/stationmeta/stationmeta.py | 5 +++--
3 files changed, 50 insertions(+), 5 deletions(-)
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index e98d180..beba349 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -932,6 +932,33 @@ class TestApps:
assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
+ def test_patch_stationmeta_roles_and_annotations(self, client, db):
+# {"annotations": [{"kind": "User",
+ response = client.patch("/stationmeta/SDZ54421?description=adding annotation text",
+ json={"stationmeta":
+ {"annotations": [{"kind": 0,
+ "text": "some annotation text",
+ "date_added": "2025-02-10 17:00",
+ "approved": True,
+ "contributor_id":1}]
+ }
+ },
+ headers={"email": "s.schroeder@fz-juelich.de"}
+ )
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'message': 'patched stationmeta record for station_id 2', 'station_id': 2}
+ response_json = response.json()
+ assert response_json == expected_resp
+ response = client.get(f"/stationmeta/id/{response_json['station_id']}")
+ response_json = response.json()
+ assert response_json['annotations'] == [{'id': 1, 'kind': 'user comment', 'text': 'some annotation text', 'date_added': '2025-02-10T17:00:00+00:00', 'approved': True, 'contributor_id': 1}]
+# assert response_json['changelog'][1]['old_value'] == ...
+# assert response_json['changelog'][1]['new_value'] == ...
+ assert response_json['changelog'][1]['author_id'] == 1
+ assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
+
+
def test_delete_roles_from_stationmeta(self, client, db):
response = client.patch("/stationmeta/delete_field/China11?field=roles",
headers={"email": "s.schroeder@fz-juelich.de"})
diff --git a/toardb/stationmeta/schemas.py b/toardb/stationmeta/schemas.py
index f544b44..fb2cd88 100644
--- a/toardb/stationmeta/schemas.py
+++ b/toardb/stationmeta/schemas.py
@@ -688,10 +688,27 @@ class StationmetaBase(StationmetaCoreBase):
def order_changelog(cls, v):
return sorted(v, key=lambda x: x.datetime)
+ @validator('roles')
+ def check_roles(cls, v):
+ if v == []:
+ return None
+ else:
+ return v
-class StationmetaPatch(StationmetaCorePatch):
- roles: List[StationmetaRolePatch] = None
- annotations: List[StationmetaAnnotationPatch] = None
+ @validator('annotations')
+ def check_annotations(cls, v):
+ if v == []:
+ return None
+ else:
+ return v
+
+
+class StationmetaPatch(StationmetaCoreCreate):
+ #roles: List[StationmetaRolePatch] = None
+ #annotations: List[StationmetaAnnotationPatch] = None
+ # just to get things working
+ roles: list = None
+ annotations: list = None
aux_images: List[StationmetaAuxImagePatch] = None
aux_docs: List[StationmetaAuxDocPatch] = None
aux_urls: List[StationmetaAuxUrlPatch] = None
diff --git a/toardb/stationmeta/stationmeta.py b/toardb/stationmeta/stationmeta.py
index 2967049..5dbf4e4 100644
--- a/toardb/stationmeta/stationmeta.py
+++ b/toardb/stationmeta/stationmeta.py
@@ -78,7 +78,8 @@ async def create_stationmeta_core(request: Request,
# 3. update
-@router.patch('/stationmeta/{station_code}', response_model=schemas.StationmetaPatch)
+#@router.patch('/stationmeta/{station_code}', response_model=schemas.StationmetaPatch)
+@router.patch('/stationmeta/{station_code}')
def patch_stationmeta_core(request: Request,
station_code: str,
stationmeta: schemas.StationmetaPatch = Body(..., embed = True),
@@ -108,7 +109,7 @@ def patch_stationmeta_core(request: Request,
@router.patch('/stationmeta/delete_field/{station_code}', response_model=schemas.StationmetaPatch)
-def patch_stationmeta_core(request: Request,
+def delete_field_from_stationmeta_core(request: Request,
station_code: str,
field: str,
access: dict = Depends(get_station_md_change_access_rights),
--
GitLab
From 0b5c8ac5bbccfe18b733239d11c016722c4809ee Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 11 Feb 2025 08:56:46 +0000
Subject: [PATCH 27/36] fixed the patching of a station's annotations
---
tests/test_stationmeta.py | 14 +++++++++-----
toardb/stationmeta/crud.py | 7 +++++++
2 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index beba349..8bdc98a 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -933,10 +933,11 @@ class TestApps:
def test_patch_stationmeta_roles_and_annotations(self, client, db):
-# {"annotations": [{"kind": "User",
response = client.patch("/stationmeta/SDZ54421?description=adding annotation text",
json={"stationmeta":
- {"annotations": [{"kind": 0,
+ {"roles": [{"role": "PointOfContact", "contact_id": 3, "status": "Active"},
+ {"role": "Originator", "contact_id": 1, "status": "Active"}],
+ "annotations": [{"kind": "User",
"text": "some annotation text",
"date_added": "2025-02-10 17:00",
"approved": True,
@@ -953,10 +954,13 @@ class TestApps:
response = client.get(f"/stationmeta/id/{response_json['station_id']}")
response_json = response.json()
assert response_json['annotations'] == [{'id': 1, 'kind': 'user comment', 'text': 'some annotation text', 'date_added': '2025-02-10T17:00:00+00:00', 'approved': True, 'contributor_id': 1}]
-# assert response_json['changelog'][1]['old_value'] == ...
-# assert response_json['changelog'][1]['new_value'] == ...
+ assert response_json['changelog'][1]['old_value'] == "{'roles': {{'role': 'ResourceProvider', 'status': 'Active', 'contact_id': 5},}"
+ assert response_json['changelog'][1]['new_value'] == (
+ "{'roles': [{'role': 'PointOfContact', 'contact_id': 3, 'status': 'Active'}, "
+ "{'role': 'Originator', 'contact_id': 1, 'status': 'Active'}], "
+ "'annotations': [{'kind': 'User', 'text': 'some annotation text', 'date_added': '2025-02-10 17:00', 'approved': True, 'contributor_id': 1}]}" )
assert response_json['changelog'][1]['author_id'] == 1
- assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
+ assert response_json['changelog'][1]['type_of_change'] == 'comprehensive metadata revision'
def test_delete_roles_from_stationmeta(self, client, db):
diff --git a/toardb/stationmeta/crud.py b/toardb/stationmeta/crud.py
index 2c1f869..a8e61f7 100644
--- a/toardb/stationmeta/crud.py
+++ b/toardb/stationmeta/crud.py
@@ -210,6 +210,11 @@ def get_unique_stationmeta_annotation(db: Session, text: str, contributor_id: in
return db_object
+# is this internal, or should this also go to public REST api?
+def get_stationmeta_annotations(db: Session, station_id: int):
+ return db.execute(select([stationmeta_core_stationmeta_annotations_table]).where(stationmeta_core_stationmeta_annotations_table.c.station_id == station_id))
+
+
def get_stationmeta_global(db: Session, station_id: int):
db_object = db.query(models.StationmetaGlobal).filter(models.StationmetaGlobal.station_id == station_id) \
.first()
@@ -243,6 +248,7 @@ def determine_stationmeta_global(db, tmp_coordinates, country):
globalmeta_dict[db_service.variable_name] = value
return globalmeta_dict
+
def create_stationmeta(db: Session, engine: Engine, stationmeta: StationmetaCreate,
author_id: int, force: bool):
stationmeta_dict = stationmeta.dict()
@@ -503,6 +509,7 @@ def patch_stationmeta(db: Session, description: str,
number_of_commas = number_of_commas - 1
for a in annotations_data:
db_annotation = models.StationmetaAnnotation(**a)
+ db_annotation.kind = get_value_from_str(toardb.toardb.AK_vocabulary,db_annotation.kind)
# check whether annotation is already present in database
db_object = get_unique_stationmeta_annotation(db, db_annotation.text, db_annotation.contributor_id)
if db_object:
--
GitLab
From 981d19ca47d90e05e75da1737f8fa43f6e0e64c9 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 11 Feb 2025 15:03:15 +0000
Subject: [PATCH 28/36] adapt endpoints to not return empty fields
---
tests/test_data.py | 48 +++++--------------------
tests/test_search.py | 11 ------
tests/test_stationmeta.py | 59 ++++++++++++++++++++++---------
tests/test_timeseries.py | 12 +------
toardb/data/data.py | 6 ++--
toardb/stationmeta/crud.py | 8 +++--
toardb/stationmeta/stationmeta.py | 4 +--
7 files changed, 62 insertions(+), 86 deletions(-)
diff --git a/tests/test_data.py b/tests/test_data.py
index c86186e..2bf517a 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -211,17 +211,14 @@ class TestApps:
'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown',
'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'roles': [],
- 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'globalmeta': None, 'annotations': [], 'changelog': []},
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'changelog': []},
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1', 'chemical_formula': 'C7H8', 'id': 7},
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''},
'roles': [{'id': 2, 'role': 'resource provider', 'status': 'active',
- 'contact': {'id': 4, 'person': None,
- 'organisation': {'id': 1, 'name': 'UBA', 'longname': 'Umweltbundesamt',
+ 'contact': {'id': 4, 'organisation': {'id': 1, 'name': 'UBA', 'longname': 'Umweltbundesamt',
'kind': 'government', 'city': 'Dessau-Roßlau', 'postcode': '06844', 'street_address': 'Wörlitzer Platz 1',
'country': 'Germany', 'homepage': 'https://www.umweltbundesamt.de', 'contact_url': 'mailto:immission@uba.de'}}}],
- 'changelog': None,
'citation': 'Umweltbundesamt: time series of toluene at Shangdianzi, accessed from the TOAR database on 2023-07-28 12:00:00'},
'data': [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
{'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'version': '1.0', 'timeseries_id': 1},
@@ -290,12 +287,9 @@ class TestApps:
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some '
'more information about the station'},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
- 'globalmeta': None,
'changelog': []},
'variable': {'name': 'toluene',
'longname': 'toluene',
@@ -313,7 +307,6 @@ class TestApps:
'role': 'resource provider',
'status': 'active',
'contact': {'id': 4,
- 'person': None,
'organisation': {'id': 1,
'name': 'UBA',
'longname': 'Umweltbundesamt',
@@ -324,7 +317,6 @@ class TestApps:
'country': 'Germany',
'homepage': 'https://www.umweltbundesamt.de',
'contact_url': 'mailto:immission@uba.de'}}}],
- 'changelog': None,
'citation': 'Umweltbundesamt: time series of toluene at '
'Shangdianzi, accessed from the TOAR database on '
'2023-07-28 12:00:00',
@@ -380,12 +372,9 @@ class TestApps:
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some '
'more information about the station'},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
- 'globalmeta': None,
'changelog': []},
'variable': {'name': 'toluene',
'longname': 'toluene',
@@ -403,7 +392,6 @@ class TestApps:
'role': 'resource provider',
'status': 'active',
'contact': {'id': 4,
- 'person': None,
'organisation': {'id': 1,
'name': 'UBA',
'longname': 'Umweltbundesamt',
@@ -414,7 +402,6 @@ class TestApps:
'country': 'Germany',
'homepage': 'https://www.umweltbundesamt.de',
'contact_url': 'mailto:immission@uba.de'}}}],
- 'changelog': None,
'citation': 'Umweltbundesamt: time series of toluene at '
'Shangdianzi, accessed from the TOAR database on '
'2023-07-28 12:00:00',
@@ -473,8 +460,8 @@ class TestApps:
'# "additional_metadata": {\n',
'# "dummy_info": "Here is some more information about the station"\n',
'# },\n',
- '# "roles": [],\n',
- '# "annotations": [],\n',
+ '# "roles": null,\n',
+ '# "annotations": null,\n',
'# "aux_images": [],\n',
'# "aux_docs": [],\n',
'# "aux_urls": [],\n',
@@ -644,16 +631,15 @@ class TestApps:
'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown',
'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'globalmeta': None, 'annotations': [], 'roles': [], 'changelog': []},
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'changelog': []},
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1', 'chemical_formula': 'C7H8', 'id': 7},
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''},
'roles': [{'id': 2, 'role': 'resource provider', 'status': 'active',
- 'contact': {'id': 4, 'person': None,
- 'organisation': {'id': 1, 'name': 'UBA', 'longname': 'Umweltbundesamt',
+ 'contact': {'id': 4, 'organisation': {'id': 1, 'name': 'UBA', 'longname': 'Umweltbundesamt',
'kind': 'government', 'city': 'Dessau-Roßlau', 'postcode': '06844', 'street_address': 'Wörlitzer Platz 1',
'country': 'Germany', 'homepage': 'https://www.umweltbundesamt.de', 'contact_url': 'mailto:immission@uba.de'}}}],
- 'changelog': None, 'citation': 'Umweltbundesamt: time series of toluene at Shangdianzi, accessed from the TOAR database on 2023-07-28 12:00:00'},
+ 'citation': 'Umweltbundesamt: time series of toluene at Shangdianzi, accessed from the TOAR database on 2023-07-28 12:00:00'},
'data': [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
{'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
{'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated verified', 'timeseries_id': 1, 'version': '1.0'},
@@ -715,12 +701,9 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
- 'globalmeta': None,
'changelog': []},
'variable': {'name': 'o3',
'longname': 'ozone',
@@ -738,7 +721,6 @@ class TestApps:
'role': 'resource provider',
'status': 'active',
'contact': {'id': 5,
- 'person': None,
'organisation': {'id': 2,
'name': 'FZJ',
'longname': 'Forschungszentrum Jülich',
@@ -749,7 +731,6 @@ class TestApps:
'country': 'Germany',
'homepage': 'https://www.fz-juelich.de',
'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
- 'changelog': None,
'citation': 'Forschungszentrum Jülich: time series of o3 at Test_China, accessed from the TOAR database on 2023-07-28 12:00:00',
'attribution': 'Test-Attributions to be announced',
'license': 'This data is published under a Creative Commons Attribution 4.0 International (CC BY 4.0). https://creativecommons.org/licenses/by/4.0/'},
@@ -795,12 +776,9 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
- 'globalmeta': None,
'changelog': []},
'variable': {'name': 'o3',
'longname': 'ozone',
@@ -818,7 +796,6 @@ class TestApps:
'role': 'resource provider',
'status': 'active',
'contact': {'id': 5,
- 'person': None,
'organisation': {'id': 2,
'name': 'FZJ',
'longname': 'Forschungszentrum Jülich',
@@ -829,7 +806,6 @@ class TestApps:
'country': 'Germany',
'homepage': 'https://www.fz-juelich.de',
'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
- 'changelog': None,
'citation': 'Forschungszentrum Jülich: time series of o3 at Test_China, accessed from the TOAR database on 2023-07-28 12:00:00',
'attribution': 'Test-Attributions to be announced',
'license': 'This data is published under a Creative Commons Attribution 4.0 International (CC BY 4.0). https://creativecommons.org/licenses/by/4.0/'},
@@ -881,12 +857,9 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
- 'globalmeta': None,
'changelog': []},
'variable': {'name': 'o3',
'longname': 'ozone',
@@ -904,7 +877,6 @@ class TestApps:
'role': 'resource provider',
'status': 'active',
'contact': {'id': 5,
- 'person': None,
'organisation':
{'id': 2,
'name': 'FZJ',
@@ -916,7 +888,6 @@ class TestApps:
'country': 'Germany',
'homepage': 'https://www.fz-juelich.de',
'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
- 'changelog': None,
'citation': 'Forschungszentrum Jülich: time series of o3 at '
'Test_China, accessed from the TOAR database on '
'2023-07-28 12:00:00',
@@ -1046,12 +1017,9 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
- 'globalmeta': None,
'changelog': []
},
'variable': {'name': 'toluene',
@@ -1069,7 +1037,7 @@ class TestApps:
'description': ''
},
'roles': [{'id': 2, 'role': 'resource provider', 'status': 'active',
- 'contact': {'id': 4, 'person': None,
+ 'contact': {'id': 4,
'organisation': {'id': 1,
'name': 'UBA',
'longname': 'Umweltbundesamt',
diff --git a/tests/test_search.py b/tests/test_search.py
index 5117b49..1163657 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -201,8 +201,6 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -297,8 +295,6 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -403,7 +399,6 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [],
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -473,7 +468,6 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [],
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -559,7 +553,6 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [],
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -629,7 +622,6 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [],
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -750,7 +742,6 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [],
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -820,7 +811,6 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [],
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -890,7 +880,6 @@ class TestApps:
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai', 'additional_metadata': {},
- 'roles': [], 'annotations': [],
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index 8bdc98a..e35338e 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -31,6 +31,14 @@ from toardb.test_base import (
get_test_engine,
test_db_session as db,
)
+from datetime import datetime
+from unittest.mock import patch
+
+# only datetime.now needs to be overridden because otherwise daterange-arguments would be provided as MagicMock-objects!
+class FixedDatetime(datetime):
+ @classmethod
+ def now(cls, tz=None):
+ return datetime(2023, 7, 28, 12, 0, 0)
class TestApps:
@@ -189,7 +197,6 @@ class TestApps:
'country': 'Germany',
'homepage': 'https://www.umweltbundesamt.de',
'contact_url': 'mailto:immission@uba.de'}}],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -275,7 +282,6 @@ class TestApps:
'country': 'Germany',
'homepage': 'https://www.fz-juelich.de',
'contact_url': 'mailto:toar-data@fz-juelich.de'}}],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -323,8 +329,6 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -375,7 +379,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [], 'annotations': [], 'aux_images': [], 'aux_docs': [],
+ 'aux_images': [], 'aux_docs': [],
'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -603,7 +607,7 @@ class TestApps:
'id': 2,
'role': 'resource provider',
'status': 'active' }],
- 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -860,6 +864,7 @@ class TestApps:
assert response_json == expected_resp
response = client.get(f"/stationmeta/id/{response_json['station_id']}")
response_json = response.json()
+ print(response_json)
# just check special changes
assert response_json['name'] == 'TTTT95TTTT'
assert response_json['changelog'][1]['old_value'] == "{'name': 'Shangdianzi'}"
@@ -968,23 +973,45 @@ class TestApps:
headers={"email": "s.schroeder@fz-juelich.de"})
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_resp = {'codes': ['China11'],
+ # Database defaults cannot be patched within pytest
+ patched_response = response.json()
+ patched_response["changelog"][-1]["datetime"] = ""
+ expected_resp = {'id': 1,
+ 'codes': ['China11'],
'name': 'Mount Tai',
'coordinates': {'lat': 36.256, 'lng': 117.106, 'alt': 1534.0},
- 'coordinate_validation_status': '0',
- 'country': '48',
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China',
'state': 'Shandong Sheng',
- 'type': '0',
- 'type_of_area': '0',
- 'timezone': '310',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
- 'globalmeta': None}
- assert response.json() == expected_resp
+ 'changelog': [
+ {
+ 'author_id': 1,
+ 'datetime': '2023-07-05T08:23:04.551645+00:00',
+ 'description': 'station created',
+ 'new_value': '',
+ 'old_value': '',
+ 'station_id': 1,
+ 'type_of_change': 'created',
+ },
+ {
+ 'author_id': 1,
+ 'datetime': '',
+ 'description': 'delete field roles',
+ 'new_value': "'roles': []",
+ 'old_value': "'roles': '[]'",
+ 'station_id': 1,
+ 'type_of_change': 'single value correction in metadata',
+ },
+ ],
+ }
+ assert patched_response == expected_resp
def test_delete_field_station_not_found(self, client, db):
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index b27d5fc..c30e6b4 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -205,8 +205,6 @@ class TestApps:
'additional_metadata': {'dummy_info': 'Here is some more '
'information about the '
'station'},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -314,8 +312,6 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -401,7 +397,6 @@ class TestApps:
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'roles': [], 'annotations': [],
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -621,8 +616,7 @@ class TestApps:
'state': 'Beijing Shi', 'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'roles': [],
- 'annotations': [], 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
'dominant_ecoregion_year2017': '-1 (undefined)',
@@ -718,8 +712,6 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -816,8 +808,6 @@ class TestApps:
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
'additional_metadata': {},
- 'roles': [],
- 'annotations': [],
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
diff --git a/toardb/data/data.py b/toardb/data/data.py
index 6f5969f..a8a893e 100644
--- a/toardb/data/data.py
+++ b/toardb/data/data.py
@@ -33,7 +33,7 @@ def get_all_data(offset: int = 0, limit: int = 100, db: Session = Depends(get_db
#get all data of one timeseries
-@router.get('/data/timeseries/{timeseries_id}', response_model=schemas.Composite, response_model_exclude_unset=True)
+@router.get('/data/timeseries/{timeseries_id}', response_model=schemas.Composite, response_model_exclude_unset=True, response_model_exclude_none=True)
def get_data(timeseries_id: int, request: Request, db: Session = Depends(get_db)):
db_data = crud.get_data(db, timeseries_id=timeseries_id, path_params=request.path_params, query_params=request.query_params)
if db_data is None:
@@ -49,7 +49,7 @@ def get_version(timeseries_id: int, request: Request, db: Session = Depends(get_
#get all data of one timeseries
-@router.get('/data/timeseries/id/{timeseries_id}', response_model=schemas.Composite, response_model_exclude_unset=True)
+@router.get('/data/timeseries/id/{timeseries_id}', response_model=schemas.Composite, response_model_exclude_unset=True, response_model_exclude_none=True)
def get_data2(timeseries_id: int, request: Request, db: Session = Depends(get_db)):
db_data = crud.get_data(db, timeseries_id=timeseries_id, path_params=request.path_params, query_params=request.query_params)
if db_data is None:
@@ -58,7 +58,7 @@ def get_data2(timeseries_id: int, request: Request, db: Session = Depends(get_db
#get all data of one timeseries (including staging data)
-@router.get('/data/timeseries_with_staging/id/{timeseries_id}', response_model=schemas.Composite, response_model_exclude_unset=True)
+@router.get('/data/timeseries_with_staging/id/{timeseries_id}', response_model=schemas.Composite, response_model_exclude_unset=True, response_model_exclude_none=True)
def get_data_with_staging(timeseries_id: int, flags: str = None, format: str = 'json', db: Session = Depends(get_db)):
db_data = crud.get_data_with_staging(db, timeseries_id=timeseries_id, flags=flags, format=format)
if db_data is None:
diff --git a/toardb/stationmeta/crud.py b/toardb/stationmeta/crud.py
index a8e61f7..2af1637 100644
--- a/toardb/stationmeta/crud.py
+++ b/toardb/stationmeta/crud.py
@@ -605,13 +605,15 @@ def delete_stationmeta_field(db: Session, station_id: int, field: str, author_id
# id can never be deleted (and of course also not changed)!!!
# there are mandatory fields (from stationmeta_core), that cannot be deleted!
# --> set these to their default value
+ new_value = ""
field_table = {'roles': stationmeta_core_stationmeta_roles_table,
'annotations': stationmeta_core_stationmeta_annotations_table}
# 'aux_images': stationmeta_core_stationmeta_aux_images_table,
# 'aux_docs': stationmeta_core_stationmeta_aux_docs_table,
# 'aux_urls': stationmeta_core_stationmeta_aux_urls_table,
- if ((field == 'roles') or (field == 'annotations')):
+ if (field in field_table):
db.execute(delete(field_table[field]).where(field_table[field].c.station_id==station_id))
+ new_value = f"'{field}': []"
# problem with automatic conversion of coordinates (although not explicitly fetched from database)
# ==> next two lines are a workaround
db_stationmeta = db.query(models.StationmetaCore).get(station_id)
@@ -622,11 +624,11 @@ def delete_stationmeta_field(db: Session, station_id: int, field: str, author_id
description=f"delete field {field}"
old_value = get_field_from_record(db, station_id, field, db_stationmeta)
db_changelog = StationmetaChangelog(description=description, station_id=station_id, author_id=author_id, type_of_change=type_of_change,
- old_value=old_value, new_value="")
+ old_value=old_value, new_value=new_value)
db.add(db_changelog)
db.commit()
# there's a mismatch with coordinates --> how to automatically switch back and forth?!
db_stationmeta.coordinates = tmp_coordinates
# hotfix: db_stationmeta.global needs to be retranslated to the dict that is understood by StationmetaGlobal
- db_stationmeta.globalmeta= None
+ db_stationmeta.globalmeta = None
return db_stationmeta
diff --git a/toardb/stationmeta/stationmeta.py b/toardb/stationmeta/stationmeta.py
index 5dbf4e4..2809f26 100644
--- a/toardb/stationmeta/stationmeta.py
+++ b/toardb/stationmeta/stationmeta.py
@@ -38,7 +38,7 @@ def get_stationmeta(station_code: str, fields: str = None, db: Session = Depends
#get all core metadata of one station (given its ID)
-@router.get('/stationmeta/id/{station_id}', response_model=schemas.Stationmeta)
+@router.get('/stationmeta/id/{station_id}', response_model=schemas.Stationmeta, response_model_exclude_none=True, response_model_exclude_unset=True)
def get_stationmeta_by_id(station_id: int, db: Session = Depends(get_db)):
db_stationmeta = crud.get_stationmeta_by_id(db, station_id=station_id)
if db_stationmeta is None:
@@ -108,7 +108,7 @@ def patch_stationmeta_core(request: Request,
raise HTTPException(status_code=401, detail="Unauthorized.")
-@router.patch('/stationmeta/delete_field/{station_code}', response_model=schemas.StationmetaPatch)
+@router.patch('/stationmeta/delete_field/{station_code}', response_model=schemas.StationmetaBase, response_model_exclude_none=True, response_model_exclude_unset=True)
def delete_field_from_stationmeta_core(request: Request,
station_code: str,
field: str,
--
GitLab
From 1d0d89dcfc94203ea329b9db0d323deb8643111c Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 11 Feb 2025 16:38:04 +0000
Subject: [PATCH 29/36] show example file for endpoint 'database_statistics'
---
static/db_statistics.json | 7 +++++++
1 file changed, 7 insertions(+)
create mode 100644 static/db_statistics.json
diff --git a/static/db_statistics.json b/static/db_statistics.json
new file mode 100644
index 0000000..32f9506
--- /dev/null
+++ b/static/db_statistics.json
@@ -0,0 +1,7 @@
+{
+ "users": 0,
+ "stations": 23979,
+ "time-series": 432880,
+ "data records": 65637800808
+}
+
--
GitLab
From d0b6e4d423d72034476ee71719c442d5544c3a21 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 11 Feb 2025 16:51:09 +0000
Subject: [PATCH 30/36] added tests for database_statistics
---
tests/test_stationmeta.py | 9 ---------
tests/test_toardb.py | 24 ++++++++++++++++++++++++
toardb/toardb.py | 4 ++--
3 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index e35338e..42701fb 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -31,14 +31,6 @@ from toardb.test_base import (
get_test_engine,
test_db_session as db,
)
-from datetime import datetime
-from unittest.mock import patch
-
-# only datetime.now needs to be overridden because otherwise daterange-arguments would be provided as MagicMock-objects!
-class FixedDatetime(datetime):
- @classmethod
- def now(cls, tz=None):
- return datetime(2023, 7, 28, 12, 0, 0)
class TestApps:
@@ -864,7 +856,6 @@ class TestApps:
assert response_json == expected_resp
response = client.get(f"/stationmeta/id/{response_json['station_id']}")
response_json = response.json()
- print(response_json)
# just check special changes
assert response_json['name'] == 'TTTT95TTTT'
assert response_json['changelog'][1]['old_value'] == "{'name': 'Shangdianzi'}"
diff --git a/tests/test_toardb.py b/tests/test_toardb.py
index 61dae73..56cb273 100644
--- a/tests/test_toardb.py
+++ b/tests/test_toardb.py
@@ -126,3 +126,27 @@ class TestApps:
[210, 'Water', '210 (Water bodies)'],
[220, 'SnowAndIce', '220 (Permanent snow and ice)']]
assert response.json() == expected_resp
+
+
+ def test_get_controlled_vocabulary_unknown_field(self, client, db):
+ response = client.get("/controlled_vocabulary/Station Landuse Type")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = "No controlled vocabulary found for 'Station Landuse Type'"
+ assert response.json() == expected_resp
+
+
+ def test_get_database_statistics(self, client, db):
+ response = client.get("/database_statistics")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = {'data records': 65637800808, 'stations': 23979, 'time-series': 432880, 'users': 0}
+ assert response.json() == expected_resp
+
+
+ def test_get_database_statistics_field(self, client, db):
+ response = client.get("/database_statistics/stations")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = 23979
+ assert response.json() == expected_resp
diff --git a/toardb/toardb.py b/toardb/toardb.py
index 962d504..a779130 100644
--- a/toardb/toardb.py
+++ b/toardb/toardb.py
@@ -76,7 +76,7 @@ async def profile_request(request: Request, call_next):
"html": HTMLRenderer,
"json": SpeedscopeRenderer,
}
- if request.query_params.get("profile", False):
+ if request.query_params.get("profile", False): # pragma: no cover
current_dir = Path(__file__).parent
profile_type = request.query_params.get("profile_format", "html")
with Profiler(interval=0.001, async_mode="enabled") as profiler:
@@ -92,7 +92,7 @@ async def profile_request(request: Request, call_next):
# check more on https://fastapi.tiangolo.com/tutorial/middleware/
@app.middleware("http")
async def add_process_time_header(request: Request, call_next):
- if request.query_params.get("timing", False):
+ if request.query_params.get("timing", False): # pragma: no cover
current_dir = Path(__file__).parent
start_time = time.time()
response = await call_next(request)
--
GitLab
From 4e3cd96f8788823fd2070ee0eee3d73f357b2cb8 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 11 Feb 2025 17:31:32 +0000
Subject: [PATCH 31/36] added pytest for geopeas_urls
---
tests/test_toardb.py | 123 +++++++++++++++++++++++++++++++++++++++++++
1 file changed, 123 insertions(+)
diff --git a/tests/test_toardb.py b/tests/test_toardb.py
index 56cb273..6f09119 100644
--- a/tests/test_toardb.py
+++ b/tests/test_toardb.py
@@ -12,12 +12,28 @@ from toardb.test_base import (
get_test_engine,
test_db_session as db,
)
+from toardb.stationmeta.models import StationmetaGlobalService
class TestApps:
def setup(self):
self.application_url = "/controlled_vocabulary"
+ @pytest.fixture(autouse=True)
+ def setup_db_data(self, db):
+ # id_seq will not be reset automatically between tests!
+ _db_conn = get_test_engine()
+ infilename = "tests/fixtures/stationmeta/stationmeta_global_services.json"
+ with open(infilename) as f:
+ metajson=json.load(f)
+ for entry in metajson:
+ new_stationmeta_global_service = StationmetaGlobalService(**entry)
+ db.add(new_stationmeta_global_service)
+ db.commit()
+ db.refresh(new_stationmeta_global_service)
+
+
+
def test_get_controlled_vocabulary(self, client, db):
response = client.get("/controlled_vocabulary")
expected_status_code = 200
@@ -150,3 +166,110 @@ class TestApps:
assert response.status_code == expected_status_code
expected_resp = 23979
assert response.json() == expected_resp
+
+
+ def test_get_geopeas_urls(self, client, db):
+ response = client.get("/geopeas_urls")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [
+ {
+ "service_url":"{base_url}/climatic_zone/?lat={lat}&lon={lon}",
+ "variable_name":"climatic_zone_year2016"
+ },
+ {
+ "service_url":"{base_url}/major_road/?lat={lat}&lon={lon}",
+ "variable_name":"distance_to_major_road_year2020"
+ },
+ {
+ "service_url":"{base_url}/population_density/?year=2015&lat={lat}&lon={lon}",
+ "variable_name":"mean_population_density_250m_year2015"
+ },
+ {
+ "service_url":"{base_url}/population_density/?year=2015&lat={lat}&lon={lon}",
+ "variable_name":"mean_population_density_5km_year2015"
+ },
+ {
+ "service_url":"{base_url}/population_density/?agg=max&radius=25000&year=2015&lat={lat}&lon={lon}",
+ "variable_name":"max_population_density_25km_year2015"
+ },
+ {
+ "service_url":"{base_url}/population_density/?agg=mean&radius=250&year=1990&lat={lat}&lon={lon}",
+ "variable_name":"mean_population_density_250m_year1990"
+ },
+ {
+ "service_url":"{base_url}/population_density/?agg=mean&radius=5000&year=1990&lat={lat}&lon={lon}",
+ "variable_name":"mean_population_density_5km_year1990"
+ },
+ {
+ "service_url":"{base_url}/population_density/?agg=max&radius=25000&year=1990&lat={lat}&lon={lon}",
+ "variable_name":"max_population_density_25km_year1990"
+ },
+ {
+ "service_url":"{base_url}/nox_emissions/?year=2015&lat={lat}&lon={lon}",
+ "variable_name":"mean_nox_emissions_10km_year2015"
+ },
+ {
+ "service_url":"{base_url}/nox_emissions/?year=2000&lat={lat}&lon={lon}",
+ "variable_name":"mean_nox_emissions_10km_year2000"
+ },
+ {
+ "service_url":"{base_url}/htap_region_tier1/?lat={lat}&country={country}",
+ "variable_name":"htap_region_tier1_year2010"
+ },
+ {
+ "service_url":"{base_url}/ecoregion/?description=false&lat={lat}&lon={lon}",
+ "variable_name":"dominant_ecoregion_year2017"
+ },
+ {
+ "service_url":"{base_url}/landcover/?year=2012&description=false&lat={lat}&lon={lon}",
+ "variable_name":"dominant_landcover_year2012"
+ },
+ {
+ "service_url":"{base_url}/ecoregion/?description=true&radius=25000&lat={lat}&lon={lon}",
+ "variable_name":"ecoregion_description_25km_year2017"
+ },
+ {
+ "service_url":"{base_url}/landcover/?year=2012&radius=25000&description=true&lat={lat}&lon={lon}",
+ "variable_name":"landcover_description_25km_year2012"
+ },
+ {
+ "service_url":"{base_url}/topography_srtm/?relative=false&lat={lat}&lon={lon}",
+ "variable_name":"mean_topography_srtm_alt_90m_year1994"
+ },
+ {
+ "service_url":"{base_url}/topography_srtm/?relative=false&agg=mean&radius=1000&lat={lat}&lon={lon}",
+ "variable_name":"mean_topography_srtm_alt_1km_year1994"
+ },
+ {
+ "service_url":"{base_url}/topography_srtm/?relative=true&agg=max&radius=5000&lat={lat}&lon={lon}",
+ "variable_name":"max_topography_srtm_relative_alt_5km_year1994"
+ },
+ {
+ "service_url":"{base_url}/topography_srtm/?relative=true&agg=min&radius=5000&lat={lat}&lon={lon}",
+ "variable_name":"min_topography_srtm_relative_alt_5km_year1994"
+ },
+ {
+ "service_url":"{base_url}/topography_srtm/?relative=true&agg=stddev&radius=5000&lat={lat}&lon={lon}",
+ "variable_name":"stddev_topography_srtm_relative_alt_5km_year1994"
+ },
+ {
+ "service_url":"{base_url}/stable_nightlights/?year=2013&lat={lat}&lon={lon}",
+ "variable_name":"mean_stable_nightlights_1km_year2013"
+ },
+ {
+ "service_url":"{base_url}/stable_nightlights/?agg=mean&radius=5000&year=2013&lat={lat}&lon={lon}",
+ "variable_name":"mean_stable_nightlights_5km_year2013"
+ },
+ {
+ "service_url":"{base_url}/stable_nightlights/?agg=max&radius=25000&year=2013&lat={lat}&lon={lon}",
+ "variable_name":"max_stable_nightlights_25km_year2013"
+ },
+ {
+ "service_url":"{base_url}/stable_nightlights/?agg=max&radius=25000&year=1992&lat={lat}&lon={lon}",
+ "variable_name":"max_stable_nightlights_25km_year1992"
+ }
+ ]
+ set_expected_resp = {json.dumps(item, sort_keys=True) for item in expected_resp}
+ set_response = {json.dumps(item, sort_keys=True) for item in response.json()}
+ assert set_response == set_expected_resp
--
GitLab
From 4076fc5cb10dac24f563fc44fa2437e866f64d40 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Wed, 12 Feb 2025 16:45:54 +0000
Subject: [PATCH 32/36] avoid an internal server error when an unknown request
id is passed to the endpoint 'request_timeseries_list_of_contributors'
---
tests/test_timeseries.py | 8 ++++++++
toardb/timeseries/crud.py | 11 ++++++++---
2 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index c30e6b4..b2711a8 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -1084,6 +1084,14 @@ class TestApps:
assert response.json() == expected_response
+ def test_request_registered_contributors_list_unknown_rid(self, client, db):
+ response = client.get("/timeseries/request_timeseries_list_of_contributors/7f0df73a-bd0f-58b9-bb17-d5cd36f89598?format=text")
+ expected_status_code = 400
+ assert response.status_code == expected_status_code
+ expected_response = 'not a registered request id: 7f0df73a-bd0f-58b9-bb17-d5cd36f89598'
+ assert response.json() == expected_response
+
+
# 3. tests updating timeseries metadata
def test_patch_timeseries_no_description(self, client, db):
diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py
index a2e7332..596b96a 100644
--- a/toardb/timeseries/crud.py
+++ b/toardb/timeseries/crud.py
@@ -618,9 +618,14 @@ def get_request_contributors(db: Session, format: str = 'text', input_handle: Up
def get_registered_request_contributors(db: Session, rid, format: str = 'text'):
- timeseries_ids = db.execute(select([s1_contributors_table]).\
- where(s1_contributors_table.c.request_id == rid)).mappings().first()['timeseries_ids']
- return get_contributors_list(db, timeseries_ids, format)
+ try:
+ timeseries_ids = db.execute(select([s1_contributors_table]).\
+ where(s1_contributors_table.c.request_id == rid)).mappings().first()['timeseries_ids']
+ return get_contributors_list(db, timeseries_ids, format)
+ except:
+ status_code=400
+ message=f"not a registered request id: {rid}"
+ return JSONResponse(status_code=status_code, content=message)
def register_request_contributors(db: Session, rid, ids):
--
GitLab
From 5b44763a82b1a64a03c6982c6ee7c9bb0ebb231b Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Sat, 15 Feb 2025 10:49:59 +0000
Subject: [PATCH 33/36] #175: additional_metadata fields are searchable
---
.../stationmeta/stationmeta_core.json | 4 +-
tests/test_data.py | 14 +-
tests/test_search.py | 180 +++++++++++++++++-
tests/test_stationmeta.py | 8 +-
tests/test_timeseries.py | 10 +-
toardb/timeseries/crud.py | 3 +-
toardb/utils/utils.py | 27 ++-
7 files changed, 219 insertions(+), 27 deletions(-)
diff --git a/tests/fixtures/stationmeta/stationmeta_core.json b/tests/fixtures/stationmeta/stationmeta_core.json
index 6120ac4..c2d1be3 100644
--- a/tests/fixtures/stationmeta/stationmeta_core.json
+++ b/tests/fixtures/stationmeta/stationmeta_core.json
@@ -8,7 +8,7 @@
"type": 0,
"type_of_area": 0,
"timezone": 310,
- "additional_metadata": {}
+ "additional_metadata": {"dummy_info": "Here is some more information about the station"}
},
{
"codes": ["SDZ54421"],
@@ -19,7 +19,7 @@
"type": 0,
"type_of_area": 0,
"timezone": 310,
- "additional_metadata": {"dummy_info": "Here is some more information about the station" }
+ "additional_metadata": {"add_type": "nature reservation"}
},
{
"codes":["China_test8"],
diff --git a/tests/test_data.py b/tests/test_data.py
index 2bf517a..b49a456 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -210,7 +210,7 @@ class TestApps:
'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi', 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown',
'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'changelog': []},
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1', 'chemical_formula': 'C7H8', 'id': 7},
@@ -285,8 +285,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some '
- 'more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -370,8 +369,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some '
- 'more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -458,7 +456,7 @@ class TestApps:
'# "type_of_area": "unknown",\n',
'# "timezone": "Asia/Shanghai",\n',
'# "additional_metadata": {\n',
- '# "dummy_info": "Here is some more information about the station"\n',
+ '# "add_type": "nature reservation"\n',
'# },\n',
'# "roles": null,\n',
'# "annotations": null,\n',
@@ -630,7 +628,7 @@ class TestApps:
'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi', 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown',
'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [], 'aux_docs': [], 'aux_urls': [], 'changelog': []},
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1', 'chemical_formula': 'C7H8', 'id': 7},
@@ -1016,7 +1014,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
diff --git a/tests/test_search.py b/tests/test_search.py
index 1163657..f129845 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -200,7 +200,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -398,7 +398,8 @@ class TestApps:
'coordinate_validation_status': 'not checked',
'country': 'China', 'state': 'Shandong Sheng',
'type': 'unknown', 'type_of_area': 'unknown',
- 'timezone': 'Asia/Shanghai', 'additional_metadata': {},
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -916,3 +917,178 @@ class TestApps:
}]},
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
assert response.json() == expected_resp
+
+
+ def test_search_with_additional_metadata(self, client, db):
+ response = client.get("/search/?additional_metadata->'absorption_cross_section'=Hearn1961")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'id': 2, 'label': 'CMA', 'order': 1,
+ 'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
+ 'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
+ 'data_origin': 'instrument', 'sampling_height': 7.0,
+ 'provider_version': 'N/A',
+ 'doi': '',
+ 'additional_metadata': {'absorption_cross_section': 'Hearn 1961',
+ 'measurement_method': 'uv_abs',
+ 'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'ebas_metadata_19740101000000_29y': {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, some long division name, SHORT, , 111 Streetname, , zipcode, Boulder, CO, USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret' } },
+ 'roles': [{'id': 1, 'role': 'resource provider', 'status': 'active',
+ 'contact': {'id': 5, 'organisation': {'id': 2, 'name': 'FZJ', 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research', 'city': 'Jülich', 'postcode': '52425', 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany', 'homepage': 'https://www.fz-juelich.de', 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
+ 'variable': {'name': 'o3', 'longname': 'ozone', 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3', 'id': 5},
+ 'station': {'id': 3, 'codes': ['China_test8'], 'name': 'Test_China',
+ 'coordinates': {'alt': 1534.0, 'lat': 36.256, 'lng': 117.106},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China', 'state': 'Shandong Sheng',
+ 'type': 'unknown', 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
+ 'aux_images': [], 'aux_docs': [], 'aux_urls': [],
+ 'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'distance_to_major_road_year2020': -999.0,
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'dominant_landcover_year2012': '10 (Cropland, rainfed)',
+ 'ecoregion_description_25km_year2017': '',
+ 'htap_region_tier1_year2010': '10 (SAF Sub Saharan/sub Sahel Africa)',
+ 'landcover_description_25km_year2012': '',
+ 'max_stable_nightlights_25km_year1992': -999.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'mean_stable_nightlights_1km_year2013': -999.0,
+ 'mean_stable_nightlights_5km_year2013': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_250m_year2015': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'mean_population_density_5km_year2015': -1.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'toar1_category': 'unclassified'},
+ 'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
+ 'description': 'station created',
+ 'old_value': '',
+ 'new_value': '',
+ 'station_id': 3,
+ 'author_id': 1,
+ 'type_of_change': 'created'
+ }]},
+ 'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
+ assert response.json() == expected_resp
+
+
+ def test_search_with_additional_metadata2(self, client, db):
+ response = client.get("/search/?additional_metadata->'absorption_cross_section'=Hearn1961"+
+ "&additional_metadata->'sampling_type'=Continuous"+
+ "&additional_metadata->'calibration_type'=Automatic")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = []
+ assert response.json() == expected_response
+
+
+ def test_search_with_additional_metadata_unknown(self, client, db):
+ response = client.get("/search/?additional_metadata->'not_yet_defined'=42")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = []
+ assert response.json() == expected_response
+
+
+ def test_search_with_additional_metadata_station(self, client, db):
+ response = client.get("/search/?station_additional_metadata->'add_type'=nature reservation")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = [{'id': 1,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata': {},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 2,
+ 'codes': ['SDZ54421'],
+ 'name': 'Shangdianzi',
+ 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China',
+ 'state': 'Beijing Shi',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {'add_type': 'nature reservation'},
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
+ 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)',
+ 'landcover_description_25km_year2012': '',
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'ecoregion_description_25km_year2017': '',
+ 'distance_to_major_road_year2020': -999.0,
+ 'mean_stable_nightlights_1km_year2013': -999.0,
+ 'mean_stable_nightlights_5km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
+ 'mean_population_density_250m_year2015': -1.0,
+ 'mean_population_density_5km_year2015': -1.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
+ 'toar1_category': 'unclassified'},
+ 'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00', 'description': 'station created', 'old_value': '', 'new_value': '', 'station_id': 2, 'author_id': 1, 'type_of_change': 'created'}]},
+ 'variable': {'name': 'toluene',
+ 'longname': 'toluene',
+ 'displayname': 'Toluene',
+ 'cf_standardname': 'mole_fraction_of_toluene_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'C7H8',
+ 'id': 7},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 4,
+ 'organisation': {'id': 1,
+ 'name': 'UBA',
+ 'longname': 'Umweltbundesamt',
+ 'kind': 'government',
+ 'city': 'Dessau-Roßlau',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'contact_url': 'mailto:immission@uba.de'}}}]}]
+ assert response.json() == expected_response
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index 42701fb..0366dd3 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -175,7 +175,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {},
+ 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
'roles': [{'id': 2,
'role': 'resource provider',
'status': 'active',
@@ -259,8 +259,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about '
- 'the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'roles': [{'id': 1,
'role': 'resource provider',
'status': 'active',
@@ -599,6 +598,7 @@ class TestApps:
'id': 2,
'role': 'resource provider',
'status': 'active' }],
+ 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -977,7 +977,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {},
+ 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index b2711a8..672d3b9 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -202,9 +202,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more '
- 'information about the '
- 'station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
@@ -396,7 +394,7 @@ class TestApps:
'country': 'China', 'state': 'Beijing Shi',
'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -615,7 +613,7 @@ class TestApps:
'coordinate_validation_status': 'not checked', 'country': 'China',
'state': 'Beijing Shi', 'type': 'unknown', 'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [], 'aux_docs': [], 'aux_urls': [],
'globalmeta': {'climatic_zone_year2016': '6 (warm temperate dry)',
'distance_to_major_road_year2020': -999.0,
@@ -711,7 +709,7 @@ class TestApps:
'type': 'unknown',
'type_of_area': 'unknown',
'timezone': 'Asia/Shanghai',
- 'additional_metadata': {'dummy_info': 'Here is some more information about the station'},
+ 'additional_metadata': {'add_type': 'nature reservation'},
'aux_images': [],
'aux_docs': [],
'aux_urls': [],
diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py
index 596b96a..cb732cd 100644
--- a/toardb/timeseries/crud.py
+++ b/toardb/timeseries/crud.py
@@ -185,7 +185,6 @@ def search_all(db, path_params, query_params, lts=False):
status_code=400
return JSONResponse(status_code=status_code, content=str(e))
lnot_role = (t_r_filter.find("NOT") > 0)
-
if fields:
# sort input fields to be sure to replace station_changelog before changelog
fields = ",".join(sorted(fields.split(','),reverse=True))
@@ -207,6 +206,8 @@ def search_all(db, path_params, query_params, lts=False):
fields2.append("timeseries.order")
elif field == "additional_metadata":
fields2.append("timeseries.additional_metadata")
+ elif field == "station_additional_metadata":
+ fields2.append("stationmeta_core.additional_metadata")
elif field == "station_id":
fields2.append("stationmeta_core.id")
elif field == "variable_id":
diff --git a/toardb/utils/utils.py b/toardb/utils/utils.py
index e84a4b2..d2da351 100644
--- a/toardb/utils/utils.py
+++ b/toardb/utils/utils.py
@@ -144,15 +144,16 @@ def create_filter(query_params, endpoint):
# determine allowed query parameters (first *only* from Timeseries)
timeseries_params = {column.name for column in inspect(Timeseries).c} | {"has_role"}
+ timeseries_params = timeseries_params | {"additional_metadata-"}
gis_params = {"bounding_box", "altitude_range"}
if endpoint in ['search', 'timeseries']:
core_params = {column.name for column in inspect(StationmetaCore).c if column.name not in ['id']}
else:
core_params = {column.name for column in inspect(StationmetaCore).c}
- core_params |= {"globalmeta"}
+ core_params |= {"globalmeta", "station_additional_metadata-"}
global_params = {column.name for column in inspect(StationmetaGlobal).c if column.name not in ['id','station_id']}
data_params = {column.name for column in inspect(Data).c} | {"daterange", "format"}
- ambig_params = {"station_id", "station_changelog", "station_country"}
+ ambig_params = {"station_id", "station_changelog", "station_country", "station_additional_metadata"}
variable_params = {column.name for column in inspect(Variable).c}
person_params = {column.name for column in inspect(Person).c}
allrel_params = {"limit", "offset", "fields", "format"}
@@ -205,12 +206,13 @@ def create_filter(query_params, endpoint):
v_filter = []
p_filter = []
# query_params is a multi-dict!
- for param in query_params:
+ for param_long in query_params:
+ param = param_long.split('>')[0]
if param not in allowed_params: #inform user, that an unknown parameter name was used (this could be a typo and falsify the result!)
raise KeyError(f"An unknown argument was received: {param}.")
if param in allrel_params or param in profiling_params:
continue
- values = [item.strip() for v in query_params.getlist(param) for item in v.split(',')]
+ values = [item.strip() for v in query_params.getlist(param_long) for item in v.split(',')]
# make sure ids are ints
if param.endswith("id"):
try:
@@ -236,6 +238,8 @@ def create_filter(query_params, endpoint):
values = translate_convoc_list(values, toardb.toardb.ST_vocabulary, "type")
elif param == "type_of_area":
values = translate_convoc_list(values, toardb.toardb.TA_vocabulary, "type of area")
+ elif param == "station_additional_metadata-":
+ param = f"{param_long[8:]}"
# exceptions for special fields (codes, name)
if param == 'codes':
tmp_filter = []
@@ -245,6 +249,10 @@ def create_filter(query_params, endpoint):
s_c_filter.append(f"({tmp_filter})")
elif param == 'name':
s_c_filter.append(f"LOWER(stationmeta_core.name) LIKE '%{values[0].lower()}%'")
+ elif param_long.split('>')[0] == "station_additional_metadata-":
+ val_mod = [ f"'\"{val}\"'::text" for val in values ]
+ values = ",".join(val_mod)
+ s_c_filter.append(f"to_json(stationmeta_core.{param})::text IN ({values})")
else:
s_c_filter.append(f"stationmeta_core.{param} IN {values}")
elif endpoint in ["stationmeta", "search"] and param in global_params:
@@ -276,6 +284,17 @@ def create_filter(query_params, endpoint):
values = translate_convoc_list(values, toardb.toardb.OT_vocabulary, "data origin type")
elif param == "data_origin":
values = translate_convoc_list(values, toardb.toardb.DO_vocabulary, "data origin")
+ elif param == "additional_metadata-":
+ param = param_long
+ if param == "additional_metadata->'absorption_cross_section'":
+ trlist = translate_convoc_list(values, toardb.toardb.CS_vocabulary, "absorption_cross_section")
+ values = [ str(val) for val in trlist ]
+ if param == "additional_metadata->'sampling_type'":
+ trlist = translate_convoc_list(values, toardb.toardb.KS_vocabulary, "sampling_type")
+ values = [ str(val) for val in trlist ]
+ if param == "additional_metadata->'calibration_type'":
+ trlist = translate_convoc_list(values, toardb.toardb.CT_vocabulary, "calibration_type")
+ values = [ str(val) for val in trlist ]
if param == "has_role":
operator = "IN"
join_operator = "OR"
--
GitLab
From cbea3147951019475d2feeb5a1efed58df28faf8 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Sat, 15 Feb 2025 13:36:42 +0000
Subject: [PATCH 34/36] made string fields from timeseries' additional_metadata
searchable
---
tests/fixtures/timeseries/timeseries.json | 2 +-
tests/test_data.py | 14 ++--
tests/test_search.py | 93 ++++++++++++++++++++++-
tests/test_timeseries.py | 12 +--
toardb/utils/utils.py | 13 +++-
5 files changed, 116 insertions(+), 18 deletions(-)
diff --git a/tests/fixtures/timeseries/timeseries.json b/tests/fixtures/timeseries/timeseries.json
index bd7c59b..47e8933 100644
--- a/tests/fixtures/timeseries/timeseries.json
+++ b/tests/fixtures/timeseries/timeseries.json
@@ -11,7 +11,7 @@
"data_origin": 0,
"data_origin_type": 0,
"sampling_height": 7,
- "additional_metadata": {},
+ "additional_metadata": {"original_units": "ppb"},
"programme_id": 0
},
{
diff --git a/tests/test_data.py b/tests/test_data.py
index b49a456..da4e3ac 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -206,7 +206,7 @@ class TestApps:
'doi': '',
'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
'data_origin': 'instrument', 'data_origin_type': 'measurement', 'provider_version': 'N/A',
- 'sampling_height': 7.0, 'additional_metadata': {},
+ 'sampling_height': 7.0, 'additional_metadata': {'original_units': 'ppb'},
'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi', 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown',
'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai',
@@ -270,7 +270,7 @@ class TestApps:
'data_origin_type': 'measurement',
'provider_version': 'N/A',
'sampling_height': 7.0,
- 'additional_metadata': {},
+ 'additional_metadata': {'original_units': 'ppb'},
'doi': '',
'coverage': -1.0,
'station': {'id': 2,
@@ -354,7 +354,7 @@ class TestApps:
'data_origin_type': 'measurement',
'provider_version': 'N/A',
'sampling_height': 7.0,
- 'additional_metadata': {},
+ 'additional_metadata': {'original_units': 'ppb'},
'doi': '',
'coverage': -1.0,
'station': {'id': 2,
@@ -433,7 +433,9 @@ class TestApps:
'# "data_origin_type": "measurement",\n',
'# "provider_version": "N/A",\n',
'# "sampling_height": 7.0,\n',
- '# "additional_metadata": {},\n',
+ '# "additional_metadata": {\n',
+ '# "original_units": "ppb"\n',
+ '# },\n',
'# "data_license_accepted": null,\n',
'# "dataset_approved_by_provider": null,\n',
'# "doi": "",\n',
@@ -624,7 +626,7 @@ class TestApps:
'license': 'This data is published under a Creative Commons Attribution 4.0 International (CC BY 4.0). https://creativecommons.org/licenses/by/4.0/',
'doi': '',
'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0, 'data_origin': 'instrument',
- 'data_origin_type': 'measurement', 'provider_version': 'N/A', 'sampling_height': 7.0, 'additional_metadata': {},
+ 'data_origin_type': 'measurement', 'provider_version': 'N/A', 'sampling_height': 7.0, 'additional_metadata': {'original_units': 'ppb'},
'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi', 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked', 'country': 'China', 'state': 'Beijing Shi', 'type': 'unknown',
'type_of_area': 'unknown', 'timezone': 'Asia/Shanghai',
@@ -1003,7 +1005,7 @@ class TestApps:
'data_origin_type': 'measurement',
'provider_version': 'N/A',
'sampling_height': 7.0,
- 'additional_metadata': {},
+ 'additional_metadata': {'original_units': 'ppb'},
'station': {'id': 2,
'codes': ['SDZ54421'],
'name': 'Shangdianzi',
diff --git a/tests/test_search.py b/tests/test_search.py
index f129845..b3c32a3 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -187,7 +187,7 @@ class TestApps:
'data_origin_type': 'measurement',
'provider_version': 'N/A',
'sampling_height': 7.0,
- 'additional_metadata': {},
+ 'additional_metadata': {"original_units": "ppb"},
'doi': '',
'coverage': -1.0,
'station': {'id': 2,
@@ -675,7 +675,7 @@ class TestApps:
assert response.status_code == expected_status_code
expected_resp = [{'id': 1,
'order': 1,
- 'additional_metadata': {},
+ 'additional_metadata': {"original_units": "ppb"},
'roles': {'role': 'resource provider', 'status': 'active', 'contact_id': 4}
},
{'id': 2,
@@ -999,6 +999,93 @@ class TestApps:
assert response.json() == expected_response
+ def test_search_with_additional_metadata3(self, client, db):
+ response = client.get("/search/?additional_metadata->'original_units'=ppb")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = [{'id': 1,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata': {"original_units": "ppb"},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 2,
+ 'codes': ['SDZ54421'],
+ 'name': 'Shangdianzi',
+ 'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China',
+ 'state': 'Beijing Shi',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {'add_type': 'nature reservation'},
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': {'mean_topography_srtm_alt_90m_year1994': -999.0,
+ 'mean_topography_srtm_alt_1km_year1994': -999.0,
+ 'max_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'min_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
+ 'climatic_zone_year2016': '6 (warm temperate dry)',
+ 'htap_region_tier1_year2010': '11 (MDE Middle East: S. Arabia, Oman, etc, Iran, Iraq)',
+ 'dominant_landcover_year2012': '11 (Cropland, rainfed, herbaceous cover)',
+ 'landcover_description_25km_year2012': '',
+ 'dominant_ecoregion_year2017': '-1 (undefined)',
+ 'ecoregion_description_25km_year2017': '',
+ 'distance_to_major_road_year2020': -999.0,
+ 'mean_stable_nightlights_1km_year2013': -999.0,
+ 'mean_stable_nightlights_5km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year2013': -999.0,
+ 'max_stable_nightlights_25km_year1992': -999.0,
+ 'mean_population_density_250m_year2015': -1.0,
+ 'mean_population_density_5km_year2015': -1.0,
+ 'max_population_density_25km_year2015': -1.0,
+ 'mean_population_density_250m_year1990': -1.0,
+ 'mean_population_density_5km_year1990': -1.0,
+ 'max_population_density_25km_year1990': -1.0,
+ 'mean_nox_emissions_10km_year2015': -999.0,
+ 'mean_nox_emissions_10km_year2000': -999.0,
+ 'toar1_category': 'unclassified'},
+ 'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00', 'description': 'station created', 'old_value': '', 'new_value': '', 'station_id': 2, 'author_id': 1, 'type_of_change': 'created'}]},
+ 'variable': {'name': 'toluene',
+ 'longname': 'toluene',
+ 'displayname': 'Toluene',
+ 'cf_standardname': 'mole_fraction_of_toluene_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'C7H8',
+ 'id': 7},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 2,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 4,
+ 'organisation': {'id': 1,
+ 'name': 'UBA',
+ 'longname': 'Umweltbundesamt',
+ 'kind': 'government',
+ 'city': 'Dessau-Roßlau',
+ 'postcode': '06844',
+ 'street_address': 'Wörlitzer Platz 1',
+ 'country': 'Germany',
+ 'homepage': 'https://www.umweltbundesamt.de',
+ 'contact_url': 'mailto:immission@uba.de'}}}]}]
+ assert response.json() == expected_response
+
+
def test_search_with_additional_metadata_unknown(self, client, db):
response = client.get("/search/?additional_metadata->'not_yet_defined'=42")
expected_status_code = 200
@@ -1022,7 +1109,7 @@ class TestApps:
'data_origin_type': 'measurement',
'provider_version': 'N/A',
'sampling_height': 7.0,
- 'additional_metadata': {},
+ 'additional_metadata': {"original_units": "ppb"},
'doi': '',
'coverage': -1.0,
'station': {'id': 2,
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index 672d3b9..6f9c527 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -189,7 +189,7 @@ class TestApps:
'data_origin_type': 'measurement',
'provider_version': 'N/A',
'sampling_height': 7.0,
- 'additional_metadata': {},
+ 'additional_metadata': {'original_units': 'ppb'},
'doi': '',
'coverage': -1.0,
'station': {'id': 2,
@@ -380,7 +380,7 @@ class TestApps:
'sampling_frequency': 'hourly', 'aggregation': 'mean', 'data_origin_type': 'measurement',
'data_start_date': '2003-09-07T15:30:00+00:00', 'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0,
'data_origin': 'instrument', 'sampling_height': 7.0,
- 'provider_version': 'N/A', 'doi': '', 'additional_metadata': {},
+ 'provider_version': 'N/A', 'doi': '', 'additional_metadata': {'original_units': 'ppb'},
'roles': [{'id': 2, 'role': 'resource provider', 'status': 'active',
'contact': {'id': 4, 'organisation': {'id': 1, 'name': 'UBA', 'longname': 'Umweltbundesamt',
'kind': 'government', 'city': 'Dessau-Roßlau', 'postcode': '06844', 'street_address': 'Wörlitzer Platz 1',
@@ -603,7 +603,7 @@ class TestApps:
'id': 1, 'label': 'CMA', 'order': 1, 'sampling_frequency': 'hourly',
'aggregation': 'mean', 'data_start_date': '2003-09-07T15:30:00+00:00',
'data_end_date': '2016-12-31T14:30:00+00:00', 'coverage': -1.0, 'data_origin': 'instrument', 'data_origin_type': 'measurement',
- 'provider_version': 'N/A', 'doi': '', 'sampling_height': 7.0, 'additional_metadata': {},
+ 'provider_version': 'N/A', 'doi': '', 'sampling_height': 7.0, 'additional_metadata': {'original_units': 'ppb'},
'roles': [{'id': 2, 'role': 'resource provider', 'status': 'active',
'contact': {'id': 4, 'organisation': {'id': 1, 'name': 'UBA', 'longname': 'Umweltbundesamt',
'kind': 'government', 'city': 'Dessau-Roßlau', 'postcode': '06844', 'street_address': 'Wörlitzer Platz 1',
@@ -653,7 +653,7 @@ class TestApps:
response = client.get("/timeseries/1?fields=additional_metadata")
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_response = {'additional_metadata': {} }
+ expected_response = {'additional_metadata': {'original_units': 'ppb'} }
assert response.json() == expected_response
@@ -662,7 +662,7 @@ class TestApps:
expected_status_code = 200
assert response.status_code == expected_status_code
expected_response = [
- {'additional_metadata': {}},
+ {'additional_metadata': {'original_units': 'ppb'}},
{'additional_metadata':
{'absorption_cross_section': 'Hearn 1961',
'ebas_metadata_19740101000000_29y':
@@ -696,7 +696,7 @@ class TestApps:
'data_origin_type': 'measurement',
'provider_version': 'N/A',
'sampling_height': 7.0,
- 'additional_metadata': {},
+ 'additional_metadata': {'original_units': 'ppb'},
'doi': '',
'coverage': -1.0,
'station': {'id': 2,
diff --git a/toardb/utils/utils.py b/toardb/utils/utils.py
index d2da351..b41ab05 100644
--- a/toardb/utils/utils.py
+++ b/toardb/utils/utils.py
@@ -289,12 +289,19 @@ def create_filter(query_params, endpoint):
if param == "additional_metadata->'absorption_cross_section'":
trlist = translate_convoc_list(values, toardb.toardb.CS_vocabulary, "absorption_cross_section")
values = [ str(val) for val in trlist ]
- if param == "additional_metadata->'sampling_type'":
+ param = f"timeseries.{param}"
+ elif param == "additional_metadata->'sampling_type'":
trlist = translate_convoc_list(values, toardb.toardb.KS_vocabulary, "sampling_type")
values = [ str(val) for val in trlist ]
- if param == "additional_metadata->'calibration_type'":
+ param = f"timeseries.{param}"
+ elif param == "additional_metadata->'calibration_type'":
trlist = translate_convoc_list(values, toardb.toardb.CT_vocabulary, "calibration_type")
values = [ str(val) for val in trlist ]
+ param = f"timeseries.{param}"
+ else:
+ val_mod = [ f"'\"{val}\"'::text" for val in values ]
+ values = "(" + ",".join(val_mod) + ")"
+ param = f"to_json(timeseries.{param})::text"
if param == "has_role":
operator = "IN"
join_operator = "OR"
@@ -311,6 +318,8 @@ def create_filter(query_params, endpoint):
t_r_filter.append(f"persons.name {operator} {values}")
t_r_filter.append(f"persons.orcid {operator} {values}")
t_r_filter = f" {join_operator} ".join(t_r_filter)
+ elif param_long.split('>')[0] == "additional_metadata-":
+ t_filter.append(f"{param} IN {values}")
else:
t_filter.append(f"timeseries.{param} IN {values}")
elif param in data_params:
--
GitLab
From 90a9e34ac203511a7e40596dcedd937339cd2482 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Sun, 16 Feb 2025 15:21:07 +0000
Subject: [PATCH 35/36] #176: toar2_category is a field of stationmeta_global
---
.../stationmeta/stationmeta_global.json | 3 ++
tests/fixtures/toardb_pytest.psql | 9 +++++
tests/test_search.py | 36 ++++++++++++-------
tests/test_stationmeta.py | 34 ++++++++++++------
tests/test_timeseries.py | 18 ++++++----
toardb/stationmeta/crud.py | 3 ++
toardb/stationmeta/models_global.py | 6 ++--
toardb/stationmeta/schemas.py | 25 +++++++++++++
8 files changed, 103 insertions(+), 31 deletions(-)
diff --git a/tests/fixtures/stationmeta/stationmeta_global.json b/tests/fixtures/stationmeta/stationmeta_global.json
index d20512d..d14ba7a 100644
--- a/tests/fixtures/stationmeta/stationmeta_global.json
+++ b/tests/fixtures/stationmeta/stationmeta_global.json
@@ -24,6 +24,7 @@
"mean_nox_emissions_10km_year2015":-999.0,
"mean_nox_emissions_10km_year2000":-999.0,
"toar1_category":0,
+ "toar2_category":1,
"station_id":1},
{"mean_topography_srtm_alt_90m_year1994":-999.0,
"mean_topography_srtm_alt_1km_year1994":-999.0,
@@ -50,6 +51,7 @@
"mean_nox_emissions_10km_year2015":-999.0,
"mean_nox_emissions_10km_year2000":-999.0,
"toar1_category":0,
+ "toar2_category":2,
"station_id":2},
{"mean_topography_srtm_alt_90m_year1994":-999.0,
"mean_topography_srtm_alt_1km_year1994":-999.0,
@@ -76,5 +78,6 @@
"mean_nox_emissions_10km_year2015":-999.0,
"mean_nox_emissions_10km_year2000":-999.0,
"toar1_category":0,
+ "toar2_category":2,
"station_id":3}
]
diff --git a/tests/fixtures/toardb_pytest.psql b/tests/fixtures/toardb_pytest.psql
index e7da295..e12ca09 100644
--- a/tests/fixtures/toardb_pytest.psql
+++ b/tests/fixtures/toardb_pytest.psql
@@ -2819,6 +2819,7 @@ CREATE TABLE IF NOT EXISTS public.stationmeta_global (
max_topography_srtm_relative_alt_5km_year1994 double precision DEFAULT '-999.0'::numeric NOT NULL,
dominant_landcover_year2012 integer DEFAULT '-1'::integer NOT NULL,
toar1_category integer DEFAULT '-1'::integer NOT NULL,
+ toar2_category integer DEFAULT '-1'::integer NOT NULL,
station_id integer NOT NULL,
min_topography_srtm_relative_alt_5km_year1994 double precision DEFAULT '-999.0'::numeric NOT NULL,
stddev_topography_srtm_relative_alt_5km_year1994 double precision DEFAULT '-999.0'::numeric NOT NULL,
@@ -3918,6 +3919,14 @@ ALTER TABLE ONLY public.stationmeta_global
ADD CONSTRAINT stationmeta_global_toar1_category_fk_tc_vocabulary_enum_val FOREIGN KEY (toar1_category) REFERENCES toar_convoc.tc_vocabulary(enum_val);
+--
+-- Name: stationmeta_global stationmeta_global_toar2_category_fk_tc_vocabulary_enum_val; Type: FK CONSTRAINT; Schema: public; Owner: postgres
+--
+
+ALTER TABLE ONLY public.stationmeta_global
+ ADD CONSTRAINT stationmeta_global_toar2_category_fk_ta_vocabulary_enum_val FOREIGN KEY (toar2_category) REFERENCES toar_convoc.ta_vocabulary(enum_val);
+
+
--
-- Name: stationmeta_roles stationmeta_roles_contact_id_fk_contacts_id; Type: FK CONSTRAINT; Schema: public; Owner: postgres
--
diff --git a/tests/test_search.py b/tests/test_search.py
index b3c32a3..17b76a7 100644
--- a/tests/test_search.py
+++ b/tests/test_search.py
@@ -228,7 +228,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00',
'description': 'station created',
'old_value': '',
@@ -322,7 +323,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -425,7 +427,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -494,7 +497,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -579,7 +583,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -648,7 +653,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -768,7 +774,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -837,7 +844,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -906,7 +914,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -976,7 +985,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -1055,7 +1065,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00', 'description': 'station created', 'old_value': '', 'new_value': '', 'station_id': 2, 'author_id': 1, 'type_of_change': 'created'}]},
'variable': {'name': 'toluene',
'longname': 'toluene',
@@ -1150,7 +1161,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00', 'description': 'station created', 'old_value': '', 'new_value': '', 'station_id': 2, 'author_id': 1, 'type_of_change': 'created'}]},
'variable': {'name': 'toluene',
'longname': 'toluene',
diff --git a/tests/test_stationmeta.py b/tests/test_stationmeta.py
index 0366dd3..c8d8614 100644
--- a/tests/test_stationmeta.py
+++ b/tests/test_stationmeta.py
@@ -241,7 +241,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'urban'},
'changelog': [{'datetime': '2023-07-05T08:23:04.551645+00:00',
'description': 'station created',
'old_value': '',
@@ -301,7 +302,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-07-15T19:27:09.463245+00:00',
'description': 'station created',
'old_value': '',
@@ -348,7 +350,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -396,7 +399,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': [{'datetime': '2023-08-15T21:16:20.596545+00:00',
'description': 'station created',
'old_value': '',
@@ -437,7 +441,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'urban'
}
}
assert response.json() == expected_resp
@@ -487,7 +492,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'urban'
}
},
{'codes': ['SDZ54421'],
@@ -515,7 +521,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'
}
},
{'codes': ['China_test8'],
@@ -543,7 +550,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'
}
}]
assert response.json() == expected_resp
@@ -647,7 +655,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'urban'},
'changelog': [{'datetime': '2023-07-05T08:23:04.551645+00:00',
'description': 'station created',
'old_value': '',
@@ -891,6 +900,7 @@ class TestApps:
json={"stationmeta":
{"globalmeta": {"climatic_zone_year2016": "WarmTemperateMoist",
"toar1_category": "RuralLowElevation",
+ "toar2_category": "Urban",
"htap_region_tier1_year2010": "HTAPTier1PAN",
"dominant_landcover_year2012": "TreeNeedleleavedEvergreenClosedToOpen",
"landcover_description_25km_year2012": "TreeNeedleleavedEvergreenClosedToOpen: 100 %",
@@ -915,7 +925,8 @@ class TestApps:
"'landcover_description_25km_year2012': '', "
"'dominant_ecoregion_year2017': 'Undefined', "
"'ecoregion_description_25km_year2017': '', "
- "'toar1_category': 'Unclassified'}" )
+ "'toar1_category': 'Unclassified', "
+ "'toar2_category': 'Suburban'}" )
assert response_json['changelog'][1]['new_value'] == (
"{'climatic_zone_year2016': 'WarmTemperateMoist', "
"'htap_region_tier1_year2010': 'HTAPTier1PAN', "
@@ -923,7 +934,8 @@ class TestApps:
"'landcover_description_25km_year2012': 'TreeNeedleleavedEvergreenClosedToOpen: 100 %', "
"'dominant_ecoregion_year2017': 'Guianansavanna', "
"'ecoregion_description_25km_year2017': 'Guianansavanna: 90 %, Miskitopineforests: 10 %'"
- ", 'toar1_category': 'RuralLowElevation'}" )
+ ", 'toar1_category': 'RuralLowElevation'"
+ ", 'toar2_category': 'Urban'}" )
assert response_json['changelog'][1]['author_id'] == 1
assert response_json['changelog'][1]['type_of_change'] == 'single value correction in metadata'
diff --git a/tests/test_timeseries.py b/tests/test_timeseries.py
index 6f9c527..76c7887 100644
--- a/tests/test_timeseries.py
+++ b/tests/test_timeseries.py
@@ -234,7 +234,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': []},
'variable': {'name': 'toluene',
'longname': 'toluene',
@@ -340,7 +341,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': []},
'variable': {'name': 'o3',
'longname': 'ozone',
@@ -420,7 +422,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': []},
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}
assert response.json() == expected_resp
@@ -639,7 +642,8 @@ class TestApps:
'mean_topography_srtm_alt_90m_year1994': -999.0,
'min_topography_srtm_relative_alt_5km_year1994': -999.0,
'stddev_topography_srtm_relative_alt_5km_year1994': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': []},
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1',
@@ -737,7 +741,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'
},
'changelog': []
},
@@ -833,7 +838,8 @@ class TestApps:
'max_population_density_25km_year1990': -1.0,
'mean_nox_emissions_10km_year2015': -999.0,
'mean_nox_emissions_10km_year2000': -999.0,
- 'toar1_category': 'unclassified'},
+ 'toar1_category': 'unclassified',
+ 'toar2_category': 'suburban'},
'changelog': []},
'variable': {'name': 'o3',
'longname': 'ozone',
diff --git a/toardb/stationmeta/crud.py b/toardb/stationmeta/crud.py
index 2af1637..e0fe66c 100644
--- a/toardb/stationmeta/crud.py
+++ b/toardb/stationmeta/crud.py
@@ -563,6 +563,9 @@ def patch_stationmeta(db: Session, description: str,
elif key == "toar1_category":
value = get_value_from_str(toardb.toardb.TC_vocabulary, value)
old_value = get_str_from_value(toardb.toardb.TC_vocabulary, old_value)
+ elif key == "toar2_category":
+ value = get_value_from_str(toardb.toardb.TA_vocabulary, value)
+ old_value = get_str_from_value(toardb.toardb.TA_vocabulary, old_value)
elif key == "htap_region_tier1_year2010":
value = get_value_from_str(toardb.toardb.TR_vocabulary, value)
old_value = get_str_from_value(toardb.toardb.TR_vocabulary, old_value)
diff --git a/toardb/stationmeta/models_global.py b/toardb/stationmeta/models_global.py
index cebd9b8..2bc3363 100644
--- a/toardb/stationmeta/models_global.py
+++ b/toardb/stationmeta/models_global.py
@@ -72,6 +72,8 @@ class StationmetaGlobal(Base):
+--------------------------------------------------+------------------------+-----------+----------+------------------------------------------------+
| toar1_category | integer | | not null | '-1'::integer |
+--------------------------------------------------+------------------------+-----------+----------+------------------------------------------------+
+ | toar2_category | integer | | not null | '0'::integer |
+ +--------------------------------------------------+------------------------+-----------+----------+------------------------------------------------+
| station_id | integer | | not null | |
+--------------------------------------------------+------------------------+-----------+----------+------------------------------------------------+
@@ -82,7 +84,6 @@ class StationmetaGlobal(Base):
"stationmeta_global_climatic_zone_check" CHECK (climatic_zone_year2016 >= 0)
"stationmeta_global_dominant_landcover_year2012_check" CHECK (dominant_landcover_year2012 >= 0)
"stationmeta_global_htap_region_tier1_check" CHECK (htap_region_tier1_year2010 >= 0)
- "stationmeta_global_toar1_category_check" CHECK (toar1_category >= 0)
Foreign-key constraints:
"stationmeta_global_station_id_29ff53dd_fk_stationmeta_core_id" FOREIGN KEY (station_id) REFERENCES stationmeta_core(id) DEFERRABLE INITIALLY DEFERRED
"stationmeta_glob_dominant_ecoregion_year2017_fk_er_voc_enum_val" FOREIGN KEY (dominant_ecoregion_year2017) REFERENCES er_vocabulary(enum_val)
@@ -90,6 +91,7 @@ class StationmetaGlobal(Base):
"stationmeta_global_climatic_zone_fk_cz_at_vocabulary_enum_val" FOREIGN KEY (climatic_zone_year2016) REFERENCES cz_vocabulary(enum_val)
"stationmeta_global_htap_region_tier1_fk_tr_vocabulary_enum_val" FOREIGN KEY (htap_region_tier1_year2010) REFERENCES tr_vocabulary(enum_val)
"stationmeta_global_toar1_category_fk_tc_vocabulary_enum_val" FOREIGN KEY (toar1_category) REFERENCES tc_vocabulary(enum_val)
+ "stationmeta_global_toar2_category_fk_tc_vocabulary_enum_val" FOREIGN KEY (toar2_category) REFERENCES ta_vocabulary(enum_val)
"""
__tablename__ = 'stationmeta_global'
# Default values do not fit CheckConstraints == > Check, how both can be done!!
@@ -97,7 +99,6 @@ class StationmetaGlobal(Base):
# CheckConstraint('climatic_zone_year2016 >= 0'),
# CheckConstraint('dominant_landcover_year2012 >= 0'),
# CheckConstraint('htap_region_tier1_year2010 >= 0'),
-# CheckConstraint('toar1_category >= 0')
# )
id = Column(Integer, STATIONMETA_GLOBAL_ID_SEQ, primary_key=True, server_default=STATIONMETA_GLOBAL_ID_SEQ.next_value())
@@ -126,6 +127,7 @@ class StationmetaGlobal(Base):
mean_nox_emissions_10km_year2015 = Column(Float(53), nullable=False, server_default=text("'-999.0'::numeric"))
mean_nox_emissions_10km_year2000 = Column(Float(53), nullable=False, server_default=text("'-999.0'::numeric"))
toar1_category = Column(ForeignKey('tc_vocabulary.enum_val'), nullable=False, server_default=text("'-1'::integer"))
+ toar2_category = Column(ForeignKey('ta_vocabulary.enum_val'), nullable=False, server_default=text("'0'::integer"))
# do not use string declaration here (not working for pytest)
# use the explicit class name here,
# see: https://groups.google.com/forum/#!topic/sqlalchemy/YjGhE4d6K4U
diff --git a/toardb/stationmeta/schemas.py b/toardb/stationmeta/schemas.py
index fb2cd88..7b53338 100644
--- a/toardb/stationmeta/schemas.py
+++ b/toardb/stationmeta/schemas.py
@@ -297,6 +297,7 @@ class StationmetaGlobalBase(BaseModel):
mean_nox_emissions_10km_year2015: float = Field(..., description="mean value within a radius of 10 km around station location of the following data of the year 2015: " + str(provenance['nox_emissions']))
mean_nox_emissions_10km_year2000: float = Field(..., description="mean value within a radius of 10 km around station location of the following data of the year 2000: " + str(provenance['nox_emissions']))
toar1_category: str = Field(..., description="The station classification for the Tropsopheric Ozone Assessment Report based on the station proxy data that are stored in the TOAR database (see controlled vocabulary: Station TOAR Category)")
+ toar2_category: str = Field(..., description="The station classification for the TOAR-II based on the station proxy data that are stored in the TOAR database and obtained from an ML approach (see controlled vocabulary: Station TOAR Category)")
# station_id: int = Field(..., description="internal station_id to which these global data belong")
@@ -308,6 +309,10 @@ class StationmetaGlobalBase(BaseModel):
def check_toar1_category(cls, v):
return tuple(filter(lambda x: x.value == int(v), toardb.toardb.TC_vocabulary))[0].display_str
+ @validator('toar2_category')
+ def check_toar2_category(cls, v):
+ return tuple(filter(lambda x: x.value == int(v), toardb.toardb.TA_vocabulary))[0].display_str
+
@validator('htap_region_tier1_year2010')
def check_htap_region_tier1(cls, v):
return tuple(filter(lambda x: x.value == int(v), toardb.toardb.TR_vocabulary))[0].display_str
@@ -355,6 +360,7 @@ class StationmetaGlobalPatch(BaseModel):
mean_nox_emissions_10km_year2015: float = None
mean_nox_emissions_10km_year2000: float = None
toar1_category: str = None
+ toar2_category: str = None
station_id: int = None
@validator('climatic_zone_year2016')
@@ -371,6 +377,13 @@ class StationmetaGlobalPatch(BaseModel):
else:
raise ValueError(f"TOAR1 category not known: {v}")
+ @validator('toar2_category')
+ def check_toar2_category(cls, v):
+ if tuple(filter(lambda x: x.string == v, toardb.toardb.TA_vocabulary)):
+ return v
+ else:
+ raise ValueError(f"TOAR2 category not known: {v}")
+
@validator('htap_region_tier1_year2010')
def check_htap_region_tier1(cls, v):
if tuple(filter(lambda x: x.string == v, toardb.toardb.TR_vocabulary)):
@@ -459,6 +472,10 @@ class StationmetaGlobalFields(StationmetaGlobalPatch):
def check_toar1_category(cls, v):
return v
+ @validator('toar2_category')
+ def check_toar2_category(cls, v):
+ return v
+
@validator('htap_region_tier1_year2010')
def check_htap_region_tier1(cls, v):
return v
@@ -497,6 +514,13 @@ class StationmetaGlobalCreate(StationmetaGlobalBase):
else:
raise ValueError(f"TOAR1 category not known: {v}")
+ @validator('toar2_category')
+ def check_toar2_category(cls, v):
+ if tuple(filter(lambda x: x.string == v, toardb.toardb.TA_vocabulary)):
+ return v
+ else:
+ raise ValueError(f"TOAR2 category not known: {v}")
+
@validator('htap_region_tier1_year2010')
def check_htap_region_tier1(cls, v):
if tuple(filter(lambda x: x.string == v, toardb.toardb.TR_vocabulary)):
@@ -551,6 +575,7 @@ class StationmetaGlobalBaseNested(StationmetaGlobalBase):
mean_nox_emissions_10km_year2015: float = None
mean_nox_emissions_10km_year2000: float = None
toar1_category: str = None
+ toar2_category: str = None
class StationmetaGlobalNestedCreate(StationmetaGlobalBaseNested):
--
GitLab
From 09b60e197b2a24c3983c3fc3bc85b1d87508ca4f Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Sun, 16 Feb 2025 15:24:08 +0000
Subject: [PATCH 36/36] #176: upload corrected fixture for toar database with
correct default value for toar2_category
---
tests/fixtures/toardb_pytest.psql | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tests/fixtures/toardb_pytest.psql b/tests/fixtures/toardb_pytest.psql
index e12ca09..02fddc8 100644
--- a/tests/fixtures/toardb_pytest.psql
+++ b/tests/fixtures/toardb_pytest.psql
@@ -2819,7 +2819,7 @@ CREATE TABLE IF NOT EXISTS public.stationmeta_global (
max_topography_srtm_relative_alt_5km_year1994 double precision DEFAULT '-999.0'::numeric NOT NULL,
dominant_landcover_year2012 integer DEFAULT '-1'::integer NOT NULL,
toar1_category integer DEFAULT '-1'::integer NOT NULL,
- toar2_category integer DEFAULT '-1'::integer NOT NULL,
+ toar2_category integer DEFAULT '0'::integer NOT NULL,
station_id integer NOT NULL,
min_topography_srtm_relative_alt_5km_year1994 double precision DEFAULT '-999.0'::numeric NOT NULL,
stddev_topography_srtm_relative_alt_5km_year1994 double precision DEFAULT '-999.0'::numeric NOT NULL,
@@ -3920,7 +3920,7 @@ ALTER TABLE ONLY public.stationmeta_global
--
--- Name: stationmeta_global stationmeta_global_toar2_category_fk_tc_vocabulary_enum_val; Type: FK CONSTRAINT; Schema: public; Owner: postgres
+-- Name: stationmeta_global stationmeta_global_toar2_category_fk_ta_vocabulary_enum_val; Type: FK CONSTRAINT; Schema: public; Owner: postgres
--
ALTER TABLE ONLY public.stationmeta_global
--
GitLab