diff --git a/toardb/contacts/contacts.py b/toardb/contacts/contacts.py
index 804c32c2cb121a7ca0e678f7c32d178294200be0..0a57194b7a103761469a8b2916cc13d5babbac1d 100644
--- a/toardb/contacts/contacts.py
+++ b/toardb/contacts/contacts.py
@@ -5,8 +5,9 @@
Simple API for contacts management
"""
-from typing import List, Union
-from fastapi import APIRouter, Depends, HTTPException, Body
+from typing import List, Union, Literal
+from fastapi import APIRouter, Depends, HTTPException, Body, Request
+
from sqlalchemy.orm import Session
from . import crud, schemas
from toardb.utils.database import ToarDbSession, get_db
@@ -18,9 +19,9 @@ router = APIRouter()
# 1. persons
#get all entries of table persons
-@router.get('/contacts/persons/', response_model=List[schemas.Person])
-def get_all_persons(offset: int = 0, limit: int = 10, db: Session = Depends(get_db)):
- persons = crud.get_all_persons(db, offset=offset, limit=limit)
+@router.get('/contacts/persons/', response_model=List[schemas.Person], response_model_exclude_none=True, response_model_exclude_unset=True)
+def get_all_persons(request: Request, db: Session = Depends(get_db)):
+ persons = crud.get_all_persons(db, path_params=request.path_params, query_params=request.query_params)
return persons
#get all metadata of one person
@@ -50,7 +51,9 @@ def create_person(person: schemas.PersonCreate = Body(..., embed = True), db: Se
#get all entries of table organisations
@router.get('/contacts/organisations/', response_model=List[schemas.Organisation])
-def get_all_organisations(offset: int = 0, limit: int = 10, db: Session = Depends(get_db)):
+def get_all_organisations(offset: int = 0, limit: int | Literal["None"] = 10, db: Session = Depends(get_db)):
+ if limit == "None":
+ limit = None
organisations = crud.get_all_organisations(db, offset=offset, limit=limit)
return organisations
@@ -79,7 +82,9 @@ def create_organisation(organisation: schemas.OrganisationCreate = Body(..., emb
# 3. combined (person + organisation)
#get all entries of table contacts
@router.get('/contacts/', response_model=List[schemas.Contact])
-def get_all_contacts(offset: int = 0, limit: int = 10, db: Session = Depends(get_db)):
+def get_all_contacts(offset: int = 0, limit: int | Literal["None"] = 10, db: Session = Depends(get_db)):
+ if limit == "None":
+ limit = None
contacts = crud.get_all_contacts(db, offset=offset, limit=limit)
return contacts
diff --git a/toardb/contacts/crud.py b/toardb/contacts/crud.py
index 6fffaaeaadbba2cfb64302b0d89ba3d9953f7b54..fc8c3b2337bb4e442665be877c6b916b35f55bd0 100644
--- a/toardb/contacts/crud.py
+++ b/toardb/contacts/crud.py
@@ -6,11 +6,12 @@ Create, Read, Update, Delete functionality
"""
+from sqlalchemy import text
from sqlalchemy.orm import Session
from fastapi.responses import JSONResponse
from . import models
from .schemas import PersonCreate, OrganisationCreate
-from toardb.utils.utils import get_value_from_str
+from toardb.utils.utils import get_value_from_str, create_filter
import toardb
@@ -48,8 +49,30 @@ def get_person(db: Session, person_id: int):
return db.query(models.Person).filter(models.Person.id == person_id).filter(models.Person.isprivate == False).first()
-def get_all_persons(db: Session, limit: int, offset: int = 0):
- return db.query(models.Person).filter(models.Person.id != 0).filter(models.Person.isprivate == False).order_by(models.Person.id).offset(offset).limit(limit).all()
+def get_all_persons(db: Session, path_params, query_params):
+ try:
+ limit, offset, fields, format, filters = create_filter(query_params, "persons")
+ p_filter = filters["p_filter"]
+ except (KeyError, ValueError) as e:
+ status_code=400
+ return JSONResponse(status_code=status_code, content=str(e))
+
+ if fields:
+ fields2 = fields.split(',')
+ db_objects_l = db.query(*map(text,fields2)).select_from(models.Person). \
+ filter(models.Person.id != 0).filter(models.Person.isprivate == False). \
+ filter(text(p_filter)). \
+ order_by(models.Person.id). \
+ limit(limit).offset(offset)
+ db_objects = []
+ for db_object_immut in db_objects_l:
+ db_object = dict(zip(fields.split(','),db_object_immut))
+ db_objects.append(db_object)
+ else:
+ db_objects = db.query(models.Person).filter(models.Person.id != 0).filter(models.Person.isprivate == False). \
+ filter(text(p_filter)). \
+ order_by(models.Person.id).offset(offset).limit(limit).all()
+ return db_objects
def get_person_by_name(db: Session, name: str):
diff --git a/toardb/contacts/schemas.py b/toardb/contacts/schemas.py
index 7d90d6ed9bd69e6ab3045856e88bae835925a8af..738b9e681d1347bd98bad66fe471b011de94b190 100644
--- a/toardb/contacts/schemas.py
+++ b/toardb/contacts/schemas.py
@@ -62,11 +62,11 @@ class Organisation(OrganisationBase):
class PersonBase(BaseModel):
id: int = Field(None, description="for internal use only")
- name: str = Field(..., description="Name of person")
- email: str = Field(..., description="Email address of person")
- phone: str = Field(..., description="Phone number of person")
+ name: str = Field(None, description="Name of person")
+ email: str = Field(None, description="Email address of person")
+ phone: str = Field(None, description="Phone number of person")
orcid: str = Field(None, description="ORCID-iD of person")
- isprivate: bool = Field(..., description="Set this flag to true if the contact details shall not be exposed publicly")
+ isprivate: bool = Field(None, description="Set this flag to true if the contact details shall not be exposed publicly")
def __str__(self):
return f"{self.name} <{self.email}>"
@@ -80,7 +80,7 @@ class PersonCreate(PersonBase):
class Person(PersonBase):
- id: int = Field(..., description="for internal use only")
+ id: int = Field(None, description="for internal use only")
class Config:
orm_mode = True
diff --git a/toardb/contacts/test_contacts.py b/toardb/contacts/test_contacts.py
index 8d63c0329d4ae94d23355d5a1d67993bf378b6f2..82dec3badf66060f27372bd209307d627414a3b2 100644
--- a/toardb/contacts/test_contacts.py
+++ b/toardb/contacts/test_contacts.py
@@ -179,11 +179,11 @@ class TestApps:
def test_get_special_person(self, client, db):
- response = client.get("/contacts/persons/id/3")
+ response = client.get("/contacts/persons/?id=3")
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_resp = {'id': 3, 'name': 'Sabine Schröder', 'email': 's.schroeder@fz-juelich.de',
- 'phone': '+49-2461-61-6397', 'orcid': '0000-0002-0309-8010', 'isprivate': False}
+ expected_resp = [{'id': 3, 'name': 'Sabine Schröder', 'email': 's.schroeder@fz-juelich.de',
+ 'phone': '+49-2461-61-6397', 'orcid': '0000-0002-0309-8010', 'isprivate': False}]
assert response.json() == expected_resp
@@ -196,11 +196,27 @@ class TestApps:
def test_get_special_person_by_name(self, client, db):
- response = client.get("/contacts/persons/Sabine Schröder")
+ response = client.get("/contacts/persons/?name=Sabine Schröder")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'id': 3, 'name': 'Sabine Schröder', 'email': 's.schroeder@fz-juelich.de',
+ 'phone': '+49-2461-61-6397', 'orcid': '0000-0002-0309-8010', 'isprivate': False}]
+ assert response.json() == expected_resp
+
+
+ def test_get_special_person_by_name_using_fields(self, client, db):
+ response = client.get("/contacts/persons/?name=Sabine Schröder&fields=orcid")
expected_status_code = 200
assert response.status_code == expected_status_code
- expected_resp = {'id': 3, 'name': 'Sabine Schröder', 'email': 's.schroeder@fz-juelich.de',
- 'phone': '+49-2461-61-6397', 'orcid': '0000-0002-0309-8010', 'isprivate': False}
+ expected_resp = [{'orcid': '0000-0002-0309-8010'}]
+ assert response.json() == expected_resp
+
+
+ def test_get_special_person_using_wrong_fieldname(self, client, db):
+ response = client.get("/contacts/persons/?name=Sabine Schröder&fields=orcid,bla")
+ expected_status_code = 400
+ assert response.status_code == expected_status_code
+ expected_resp = "Wrong field given: bla"
assert response.json() == expected_resp
diff --git a/toardb/data/crud.py b/toardb/data/crud.py
index a0c1525743101dd387d4f91ebceef5b61513fd6b..792664b1c2ce85fda4ef9d5395e2160a72d0245c 100644
--- a/toardb/data/crud.py
+++ b/toardb/data/crud.py
@@ -125,7 +125,8 @@ def get_data(db: Session, timeseries_id: int, path_params, query_params):
flags = ",".join([item.strip() for v in query_params.getlist("flags") for item in v.split(',')])
if not flags:
flags = None
- limit, offset, fields, format, t_filter, t_r_filter, s_c_filter, s_g_filter, d_filter = create_filter(query_params, "data")
+ limit, offset, fields, format, filters = create_filter(query_params, "data")
+ d_filter = filters["d_filter"]
except KeyError as e:
status_code=400
return JSONResponse(status_code=status_code, content=str(e))
@@ -180,9 +181,10 @@ def get_data_with_staging(db: Session, timeseries_id: int, flags: str, format: s
dummy = record.variable
dummy = record.programme
dummy = record.changelog
- attribution, citation = get_citation(db, timeseries_id=timeseries_id).values()
+ attribution, citation, license_txt = get_citation(db, timeseries_id=timeseries_id).values()
record_dict = record.__dict__
record_dict['citation'] = citation
+ record_dict['license'] =license_txt
if attribution != None:
record_dict['attribution'] = attribution
return TimeseriesWithCitation(**record_dict)
diff --git a/toardb/data/toarqc_config/leaf_area_index_realtime.json b/toardb/data/toarqc_config/leaf_area_index_realtime.json
new file mode 100644
index 0000000000000000000000000000000000000000..6cdb18dbd596da7ea8c239e64352ecfd86712551
--- /dev/null
+++ b/toardb/data/toarqc_config/leaf_area_index_realtime.json
@@ -0,0 +1,9 @@
+[
+ {
+ "range_test":
+ {
+ "low_thres": 0,
+ "high_thres": 7
+ }
+ }
+]
diff --git a/toardb/data/toarqc_config/leaf_area_index_realtime.yaml b/toardb/data/toarqc_config/leaf_area_index_realtime.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..62bcfb5603b4ffefeb94e2760bbc8444f9db4ca4
--- /dev/null
+++ b/toardb/data/toarqc_config/leaf_area_index_realtime.yaml
@@ -0,0 +1,5 @@
+---
+- range_test:
+ low_thres: 0
+ high_thres: 7
+...
diff --git a/toardb/data/toarqc_config/leaf_area_index_standard.json b/toardb/data/toarqc_config/leaf_area_index_standard.json
new file mode 100644
index 0000000000000000000000000000000000000000..6cdb18dbd596da7ea8c239e64352ecfd86712551
--- /dev/null
+++ b/toardb/data/toarqc_config/leaf_area_index_standard.json
@@ -0,0 +1,9 @@
+[
+ {
+ "range_test":
+ {
+ "low_thres": 0,
+ "high_thres": 7
+ }
+ }
+]
diff --git a/toardb/data/toarqc_config/leaf_area_index_standard.yaml b/toardb/data/toarqc_config/leaf_area_index_standard.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..62bcfb5603b4ffefeb94e2760bbc8444f9db4ca4
--- /dev/null
+++ b/toardb/data/toarqc_config/leaf_area_index_standard.yaml
@@ -0,0 +1,5 @@
+---
+- range_test:
+ low_thres: 0
+ high_thres: 7
+...
diff --git a/toardb/data/toarqc_config/tdew2m_realtime.json b/toardb/data/toarqc_config/tdew2m_realtime.json
new file mode 100644
index 0000000000000000000000000000000000000000..cbf69ffd76a7f3cb536644924f8591b6422a2a30
--- /dev/null
+++ b/toardb/data/toarqc_config/tdew2m_realtime.json
@@ -0,0 +1,9 @@
+[
+ {
+ "range_test":
+ {
+ "low_thres": -85,
+ "high_thres": 60
+ }
+ }
+]
diff --git a/toardb/data/toarqc_config/tdew2m_realtime.yaml b/toardb/data/toarqc_config/tdew2m_realtime.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..359dfde66d10b71c80817d1a1074b1a07de6abf5
--- /dev/null
+++ b/toardb/data/toarqc_config/tdew2m_realtime.yaml
@@ -0,0 +1,5 @@
+---
+- range_test:
+ low_thres: -85
+ high_thres: 60
+...
diff --git a/toardb/data/toarqc_config/tdew2m_standard.json b/toardb/data/toarqc_config/tdew2m_standard.json
new file mode 100644
index 0000000000000000000000000000000000000000..cbf69ffd76a7f3cb536644924f8591b6422a2a30
--- /dev/null
+++ b/toardb/data/toarqc_config/tdew2m_standard.json
@@ -0,0 +1,9 @@
+[
+ {
+ "range_test":
+ {
+ "low_thres": -85,
+ "high_thres": 60
+ }
+ }
+]
diff --git a/toardb/data/toarqc_config/tdew2m_standard.yaml b/toardb/data/toarqc_config/tdew2m_standard.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..359dfde66d10b71c80817d1a1074b1a07de6abf5
--- /dev/null
+++ b/toardb/data/toarqc_config/tdew2m_standard.yaml
@@ -0,0 +1,5 @@
+---
+- range_test:
+ low_thres: -85
+ high_thres: 60
+...
diff --git a/toardb/stationmeta/crud.py b/toardb/stationmeta/crud.py
index 9b154f328ed5f818f29d0172a8f5d88102417540..e1dbe7addb42c7f2b1e274e5f9b43a44b13e869f 100644
--- a/toardb/stationmeta/crud.py
+++ b/toardb/stationmeta/crud.py
@@ -102,8 +102,10 @@ def get_all_stationmeta_core(db: Session, limit: int, offset: int = 0):
# same method as above (is the above still needed?)
def get_all_stationmeta(db: Session, path_params, query_params):
try:
- limit, offset, fields, format, t_filter, t_r_filter, s_c_filter, s_g_filter, d_filter = create_filter(query_params, "stationmeta")
- except KeyError as e:
+ limit, offset, fields, format, filters = create_filter(query_params, "stationmeta")
+ s_c_filter = filters["s_c_filter"]
+ s_g_filter = filters["s_g_filter"]
+ except (KeyError, ValueError) as e:
status_code=400
return JSONResponse(status_code=status_code, content=str(e))
diff --git a/toardb/timeseries/crud.py b/toardb/timeseries/crud.py
index e55274f4cd554b3c3ecc6efbe4d4f0e833d88ff4..3e3c792b3091d1ea398bddc0555ec90c484c38b8 100644
--- a/toardb/timeseries/crud.py
+++ b/toardb/timeseries/crud.py
@@ -27,6 +27,26 @@ from toardb.utils.utils import get_value_from_str, get_str_from_value, create_fi
import toardb
+def clean_additional_metadata(ad_met_dict):
+ if not isinstance(ad_met_dict,dict):
+ tmp = ad_met_dict.replace('"','\\"')
+ return tmp.replace("'",'"')
+ # there is a mismatch with additional_metadata
+ additional_metadata = ad_met_dict
+ for key, value in additional_metadata.items():
+ if isinstance(value,dict):
+ for key2, value2 in value.items():
+ if isinstance(value2,str):
+ additional_metadata[key][key2] = value2.replace("'","$apostroph$")
+ else:
+ if isinstance(value,str):
+ additional_metadata[key] = value.replace("'","$apostroph$")
+ additional_metadata = str(additional_metadata).replace('"','\\"')
+ additional_metadata = str(additional_metadata).replace("'",'"')
+ additional_metadata = str(additional_metadata).replace("$apostroph$","'")
+ return additional_metadata
+
+
def get_timeseries(db: Session, timeseries_id: int, fields: str = None):
if fields:
fields = ','.join('"{}"'.format(word) for word in fields.split(','))
@@ -35,7 +55,7 @@ def get_timeseries(db: Session, timeseries_id: int, fields: str = None):
resultproxy = db.execute(f'SELECT {fields} FROM timeseries WHERE id={timeseries_id} LIMIT 1')
db_object_dict = [ rowproxy._asdict() for rowproxy in resultproxy ][0]
if fields.find('additional_metadata') != -1:
- db_object_dict['additional_metadata'] = str(db_object_dict['additional_metadata']).replace("'",'"')
+ db_object_dict['additional_metadata'] = clean_additional_metadata(db_object_dict['additional_metadata'])
db_object = models.Timeseries(**db_object_dict)
else:
db_object = db.query(models.Timeseries).filter(models.Timeseries.id == timeseries_id).first()
@@ -48,14 +68,14 @@ def get_timeseries(db: Session, timeseries_id: int, fields: str = None):
if db_object:
try:
# there is a mismatch with additional_metadata
- db_object.additional_metadata = str(db_object.additional_metadata).replace("'",'"')
+ db_object.additional_metadata = clean_additional_metadata(db_object.additional_metadata)
except:
pass
try:
# there is also a mismatch with coordinates and additional_metadata from station object
if isinstance(db_object.station.coordinates, (WKBElement, WKTElement)):
db_object.station.coordinates = get_coordinates_from_geom(db_object.station.coordinates)
- db_object.station.additional_metadata = str(db_object.station.additional_metadata).replace("'",'"')
+ db_object.station.additional_metadata = clean_additional_metadata(db_object.station.additional_metadata)
except:
pass
return db_object
@@ -70,16 +90,22 @@ def get_citation(db: Session, timeseries_id: int, datetime: dt.datetime = None):
db_object = db.query(models.Timeseries).filter(models.Timeseries.id == timeseries_id).first()
# there is a mismatch with additional_metadata
PI = "unknown"
+ originators = False
attribution = None
if db_object:
pi_role = get_value_from_str(toardb.toardb.RC_vocabulary,'PrincipalInvestigator')
originator_role = get_value_from_str(toardb.toardb.RC_vocabulary,'Originator')
+ contributor_role = get_value_from_str(toardb.toardb.RC_vocabulary,'Contributor')
list_of_originators = []
for db_role in db_object.roles:
if (db_role.role == pi_role):
db_contact = get_contact(db, contact_id = db_role.contact_id)
PI = db_contact.name
elif (db_role.role == originator_role):
+ originators = True
+ db_contact = get_contact(db, contact_id = db_role.contact_id)
+ list_of_originators.append(db_contact.name)
+ elif (db_role.role == contributor_role):
db_contact = get_contact(db, contact_id = db_role.contact_id)
list_of_originators.append(db_contact.name)
list_of_data_originators = ", ".join(list_of_originators)
@@ -97,7 +123,7 @@ def get_citation(db: Session, timeseries_id: int, datetime: dt.datetime = None):
dataset_version = db_object.provider_version
citation = f"{PI}: time series of {var} at {station}, accessed from the TOAR database on {datetime}"
if attribution and list_of_data_originators:
- attribution = attribution.format(list_of_data_originators=list_of_data_originators)
+ attribution = attribution.format(orga_or_origs="data originators" if originators else "contributing organisations", list_of_data_originators=list_of_data_originators)
if dataset_version != 'N/A':
citation += f", original dataset version {dataset_version}"
license_txt = "This data is published under a Creative Commons Attribution 4.0 International (CC BY 4.0). https://creativecommons.org/licenses/by/4.0/"
@@ -111,7 +137,7 @@ def adapt_db_object(fields, fields1, db_object_immut):
except:
pass
try:
- db_object['additional_metadata'] = str(db_object['additional_metadata']).replace("'",'"')
+ db_object['additional_metadata'] = clean_additional_metadata(db_object['additional_metadata'])
except:
pass
if "changelog" in fields:
@@ -126,8 +152,12 @@ def adapt_db_object(fields, fields1, db_object_immut):
def search_all(db, path_params, query_params):
try:
- limit, offset, fields, format, t_filter, t_r_filter, s_c_filter, s_g_filter, d_filter = create_filter(query_params, "search")
- except KeyError as e:
+ limit, offset, fields, format, filters = create_filter(query_params, "search")
+ t_filter = filters["t_filter"]
+ t_r_filter = filters["t_r_filter"]
+ s_c_filter = filters["s_c_filter"]
+ s_g_filter = filters["s_g_filter"]
+ except (KeyError, ValueError) as e:
status_code=400
return JSONResponse(status_code=status_code, content=str(e))
@@ -208,12 +238,12 @@ def search_all(db, path_params, query_params):
order_by(models.Timeseries.id). \
limit(limit).offset(offset).all()
for db_object in db_objects:
- # there is a mismatch with additional_metadata
- db_object.additional_metadata = str(db_object.additional_metadata).replace("'",'"')
# there is also a mismatch with coordinates and additional_metadata from station object
if isinstance(db_object.station.coordinates, (WKBElement, WKTElement)):
db_object.station.coordinates = get_coordinates_from_geom(db_object.station.coordinates)
- db_object.station.additional_metadata = str(db_object.station.additional_metadata).replace("'",'"')
+ # there is a mismatch with additional_metadata
+ db_object.station.additional_metadata = clean_additional_metadata(db_object.station.additional_metadata)
+ db_object.additional_metadata = clean_additional_metadata(db_object.additional_metadata)
# only for internal use!
del db_object.data_license_accepted
del db_object.dataset_approved_by_provider
@@ -223,24 +253,40 @@ def search_all(db, path_params, query_params):
#def get_all_timeseries(db: Session, limit: int, offset: int, station_code: str):
def get_all_timeseries(db, path_params, query_params):
try:
- limit, offset, fields, format, t_filter, tr_filter, s_c_filter, s_g_filter, d_filter = create_filter(query_params, "timeseries")
- except KeyError as e:
+ limit, offset, fields, format, filters = create_filter(query_params, "timeseries")
+ t_filter = filters["t_filter"]
+ s_c_filter = filters["s_c_filter"]
+ except (KeyError, ValueError) as e:
status_code=400
return JSONResponse(status_code=status_code, content=str(e))
- db_objects = db.query(models.Timeseries).filter(text(t_filter)).order_by(models.Timeseries.id). \
- limit(limit).offset(offset).all()
- for db_object in db_objects:
- # there is a mismatch with additional_metadata
- db_object.additional_metadata = str(db_object.additional_metadata).replace("'",'"')
- # there is also a mismatch with coordinates and additional_metadata from station object
- if isinstance(db_object.station.coordinates, (WKBElement, WKTElement)):
- db_object.station.coordinates = get_coordinates_from_geom(db_object.station.coordinates)
- db_object.station.additional_metadata = str(db_object.station.additional_metadata).replace("'",'"')
- # only for internal use!
- del db_object.data_license_accepted
- del db_object.dataset_approved_by_provider
- return db_objects
+ if fields:
+ fields2 = fields.split(',')
+ db_objects_l = db.query(*map(text,fields2)).select_from(models.Timeseries).filter(text(s_c_filter)). \
+ order_by(models.Timeseries.id). \
+ limit(limit).offset(offset)
+ db_objects = []
+ for db_object_immut in db_objects_l:
+ db_object = dict(zip(fields.split(','),db_object_immut))
+ db_objects.append(db_object)
+ else:
+ db_objects = db.query(models.Timeseries).filter(text(t_filter)).order_by(models.Timeseries.id). \
+ limit(limit).offset(offset).all()
+ for db_object in db_objects:
+ # there is also a mismatch with coordinates and additional_metadata from station object
+ if isinstance(db_object.station.coordinates, (WKBElement, WKTElement)):
+ db_object.station.coordinates = get_coordinates_from_geom(db_object.station.coordinates)
+ # there is a mismatch with additional_metadata
+ db_object.additional_metadata = clean_additional_metadata(db_object.additional_metadata)
+ db_object.station.additional_metadata = clean_additional_metadata(db_object.station.additional_metadata)
+ # only for internal use!
+ del db_object.data_license_accepted
+ del db_object.dataset_approved_by_provider
+
+ if limit:
+ return db_objects[:limit]
+ else:
+ return db_objects
def get_timeseries_by_unique_constraints(db: Session, station_id: int, variable_id: int, resource_provider: str = None,
@@ -259,6 +305,11 @@ def get_timeseries_by_unique_constraints(db: Session, station_id: int, variable_
Criterion 14.9: data filtering procedures or other special dataset identifiers (use database field 'label')
"""
+# print("in get_timeseries_by_unique_constraints")
+# print(f"station_id: {station_id}, variable_id: {variable_id}, resource_provider: {resource_provider}, ", \
+# f"sampling_frequency: {sampling_frequency}, provider_version: {provider_version}, data_origin_type: {data_origin_type}, ", \
+# f"data_origin: {sampling_frequency}, sampling_height: {sampling_height}, label: {label}")
+
# filter for criterion 14.1 and 14.2
ret_db_object = db.query(models.Timeseries).filter(models.Timeseries.station_id == station_id) \
.filter(models.Timeseries.variable_id == variable_id).all()
@@ -280,7 +331,7 @@ def get_timeseries_by_unique_constraints(db: Session, station_id: int, variable_
for role in db_object.roles:
# resource provider is always an organisation!
organisation = get_contact(db, contact_id=role.contact_id)
- if ((organisation.longname == resource_provider) and (role_num == role.role)):
+ if ((role_num == role.role) and (organisation.longname == resource_provider)):
found = True
if not found:
ret_db_object.pop(counter)
@@ -402,11 +453,11 @@ def get_timeseries_by_unique_constraints(db: Session, station_id: int, variable_
if len(ret_db_object) == 1:
ret_db_object = ret_db_object[0]
# there is a mismatch with additional_metadata
- ret_db_object.additional_metadata = str(ret_db_object.additional_metadata).replace("'",'"')
+ ret_db_object.additional_metadata = clean_additional_metadata(ret_db_object.additional_metadata)
# there is also a mismatch with coordinates and additional_metadata from station object
if isinstance(ret_db_object.station.coordinates, (WKBElement, WKTElement)):
ret_db_object.station.coordinates = get_coordinates_from_geom(ret_db_object.station.coordinates)
- ret_db_object.station.additional_metadata = str(ret_db_object.station.additional_metadata).replace("'",'"')
+ ret_db_object.station.additional_metadata = clean_additional_metadata(ret_db_object.station.additional_metadata)
else:
status_code=405
message=f"Timeseries not unique, more criteria need to be defined."
@@ -474,6 +525,12 @@ def create_timeseries(db: Session, timeseries: TimeseriesCreate):
message=f"Station (station_id: {timeseries.station_id}) not found in database."
return JSONResponse(status_code=status_code, content=message)
if timeseries_dict['additional_metadata']:
+ for key, value in timeseries_dict['additional_metadata'].items():
+ if isinstance(value,dict):
+ for key2, value2 in value.items():
+ timeseries_dict['additional_metadata'][key][key2] = value2.replace("''","'")
+ else:
+ timeseries_dict['additional_metadata'][key] = value.replace("''","'")
if 'absorption_cross_section' in timeseries_dict['additional_metadata']:
value = get_value_from_str(toardb.toardb.CS_vocabulary,timeseries_dict['additional_metadata']['absorption_cross_section'])
timeseries_dict['additional_metadata']['absorption_cross_section'] = value
@@ -519,7 +576,8 @@ def create_timeseries(db: Session, timeseries: TimeseriesCreate):
# there is a mismatch with additional_metadata
# in upload command, we have now: "additional_metadata": "{}"
# but return from this method gives (=database): "additional_metadata": {}
- db_timeseries.additional_metadata = json.loads(str(db_timeseries.additional_metadata).replace("'",'"'))
+# print(db_timeseries.additional_metadata)
+# db_timeseries.additional_metadata = json.loads(str(db_timeseries.additional_metadata).replace("'",'"'))
db_timeseries.sampling_frequency = get_value_from_str(toardb.toardb.SF_vocabulary,db_timeseries.sampling_frequency)
db_timeseries.aggregation = get_value_from_str(toardb.toardb.AT_vocabulary,db_timeseries.aggregation)
db_timeseries.data_origin_type = get_value_from_str(toardb.toardb.OT_vocabulary,db_timeseries.data_origin_type)
@@ -581,6 +639,12 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
# check for controlled vocabulary in additional metadata
# do this in dictionary that always contains additional_metadata entry
if timeseries_dict['additional_metadata']:
+ for key, value in timeseries_dict['additional_metadata'].items():
+ if isinstance(value,dict):
+ for key2, value2 in value.items():
+ timeseries_dict['additional_metadata'][key][key2] = value2.replace("''","'")
+ else:
+ timeseries_dict['additional_metadata'][key] = value.replace("''","'")
if 'absorption_cross_section' in timeseries_dict['additional_metadata']:
value = get_value_from_str(toardb.toardb.CS_vocabulary,timeseries_dict['additional_metadata']['absorption_cross_section'])
timeseries_dict['additional_metadata']['absorption_cross_section'] = value
@@ -593,14 +657,18 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
# delete empty fields from timeseries_dict already at this place, to be able to
# distinguish between "single value correction in metadata" and "comprehensive metadata revision"
# (see controlled vocabulary "CL_vocabulary")
+ roles_data = timeseries_dict.pop('roles', None)
+ annotations_data = timeseries_dict.pop('annotations', None)
timeseries_dict2 = {k: v for k, v in timeseries_dict.items() if v is not None}
number_of_elements = len(timeseries_dict2)
+ if roles_data:
+ number_of_elements +=1
+ if annotations_data:
+ number_of_elements +=1
if (number_of_elements == 1):
type_of_change = get_value_from_str(toardb.toardb.CL_vocabulary,"SingleValue")
else:
type_of_change = get_value_from_str(toardb.toardb.CL_vocabulary,"Comprehensive")
- roles_data = timeseries_dict.pop('roles', None)
- annotations_data = timeseries_dict.pop('annotations', None)
db_obj = models.Timeseries(**timeseries_dict2)
# also the sqlalchemy get will call get_timeseries here!!!
# --> therefore call it right away
@@ -617,15 +685,10 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
db_timeseries.station.coordinates = get_geom_from_coordinates(db_timeseries.station.coordinates)
except:
pass
- try:
- db_timeseries.station.additional_metadata = json.loads(str(db_timeseries.station.additional_metadata).replace("'",'"'))
- except:
- pass
- # also problems with additional metadata (correctly reported in changelog)...
- try:
- db_timeseries.additional_metadata = json.loads(db_timeseries.additional_metadata)
- except:
- pass
+# try:
+# db_timeseries.station.additional_metadata = json.loads(str(db_timeseries.station.additional_metadata).replace("'",'"'))
+# except:
+# pass
# prepare changelog entry/entries
no_log = (description == 'NOLOG')
if not no_log:
@@ -641,8 +704,8 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
# there is a mismatch with additional_metadata
# in upload command, we have now: "additional_metadata": "{}"
# but return from this method gives (=database): "additional_metadata": {}
- if timeseries_dict['additional_metadata']:
- db_timeseries.additional_metadata = json.loads(str(timeseries_dict['additional_metadata']).replace("'",'"'))
+# if timeseries_dict['additional_metadata']:
+# db_timeseries.additional_metadata = clean_additional_metadata(db_timeseries.additional_metadata)
db.add(db_timeseries)
result = db.commit()
# store roles and update association table
@@ -661,7 +724,7 @@ def patch_timeseries(db: Session, description: str, timeseries_id: int, timeseri
old_roles.append(old_value)
old_values['roles'] = old_roles
for r in roles_data:
- db_role = models.StationmetaRole(**r)
+ db_role = models.TimeseriesRole(**r)
db_role.role = get_value_from_str(toardb.toardb.RC_vocabulary,db_role.role)
db_role.status = get_value_from_str(toardb.toardb.RS_vocabulary,db_role.status)
# check whether role is already present in database
diff --git a/toardb/timeseries/fixtures/timeseries.json b/toardb/timeseries/fixtures/timeseries.json
index 99e0649c1705b31495819782dba833985fb57a2c..7c4cfc7d053fa64a0e349b1558434f51485c2579 100644
--- a/toardb/timeseries/fixtures/timeseries.json
+++ b/toardb/timeseries/fixtures/timeseries.json
@@ -25,6 +25,6 @@
"data_origin": 0,
"data_origin_type": 0,
"sampling_height": 7,
- "additional_metadata": "{}"
+ "additional_metadata": "{\"absorption_cross_section\": 0}"
}
]
diff --git a/toardb/timeseries/schemas.py b/toardb/timeseries/schemas.py
index 2b7819969c5866c89b3cd3c640d001ed3f02d968..c657ec4d16aacf5d29c742c9f5c73fcbebc70546 100644
--- a/toardb/timeseries/schemas.py
+++ b/toardb/timeseries/schemas.py
@@ -121,9 +121,9 @@ class TimeseriesCore(TimeseriesCoreBase):
class TimeseriesRoleBase(BaseModel):
id: int = None
- role: str = Field(..., description="Role of contact (see controlled vocabulary: Role Code)")
- status: str = Field(..., description="Status of contact (see controlled vocabulary: Role Status)")
- contact: Contact = Field(..., description="Contact for this role")
+ role: str = Field(None, description="Role of contact (see controlled vocabulary: Role Code)")
+ status: str = Field(None, description="Status of contact (see controlled vocabulary: Role Status)")
+ contact: Contact = Field(None, description="Contact for this role")
@validator('role')
def check_role(cls, v):
@@ -317,6 +317,8 @@ class TimeseriesPatch(TimeseriesCoreCreate):
data_end_date: dt.datetime = None
sampling_height: float = None
# roles: List[TimeseriesRole] = None
+ # just to get things working
+ roles: list = None
# annotations: List[TimeseriesAnnotation] = None
# variable: Variable = None
# station: StationmetaCoreBase = None
@@ -335,8 +337,8 @@ class Timeseries(TimeseriesBase):
# hot fix
coordinates: Coordinates = None
name: str = None
- variable_id: int = None
- station_id: int = None
+# variable_id: int = None
+# station_id: int = None
codes: List[str] = None
station_country: str = None
type: str = None
diff --git a/toardb/timeseries/test_search.py b/toardb/timeseries/test_search.py
index 6a9c32bbd0076aa14ff9fb68c18af1a594d62155..8a4ae679e79a512a384e259a79ce544555d7f4cf 100644
--- a/toardb/timeseries/test_search.py
+++ b/toardb/timeseries/test_search.py
@@ -181,7 +181,6 @@ class TestApps:
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1',
'chemical_formula': 'C7H8', 'id': 7},
- 'variable_id': 7,
'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi',
'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked',
@@ -220,6 +219,5 @@ class TestApps:
'toar1_category': 'unclassified',
'wheat_production_year2000': -999.0},
'changelog': []},
- 'station_id': 2,
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}]
assert response.json() == expected_resp
diff --git a/toardb/timeseries/test_timeseries.py b/toardb/timeseries/test_timeseries.py
index 3ee64a4a5efb06a9ae65cb429455462d4e85d2f4..0559a0575670b1577c86391239876e8a1b082e1b 100644
--- a/toardb/timeseries/test_timeseries.py
+++ b/toardb/timeseries/test_timeseries.py
@@ -181,7 +181,6 @@ class TestApps:
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1',
'chemical_formula': 'C7H8', 'id': 7},
- 'variable_id': 7,
'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi',
'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked',
@@ -220,9 +219,8 @@ class TestApps:
'toar1_category': 'unclassified',
'wheat_production_year2000': -999.0},
'changelog': []},
- 'station_id': 2,
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}},
- {'additional_metadata': {},
+ {'additional_metadata': {'absorption_cross_section': 'Hearn 1961'},
'aggregation': 'mean',
'data_end_date': '2016-12-31T14:30:00+00:00',
'data_origin': 'instrument',
@@ -290,7 +288,6 @@ class TestApps:
'timezone': 'Asia/Shanghai',
'type': 'unknown',
'type_of_area': 'unknown'},
- 'station_id': 3,
'variable': {'cf_standardname': 'mole_fraction_of_ozone_in_air',
'chemical_formula': 'O3',
'displayname': 'Ozone',
@@ -298,7 +295,7 @@ class TestApps:
'longname': 'ozone',
'name': 'o3',
'units': 'nmol mol-1'},
- 'variable_id': 5}]
+ }]
assert response.json() == expected_resp
@@ -318,7 +315,6 @@ class TestApps:
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1',
'chemical_formula': 'C7H8', 'id': 7},
- 'variable_id': 7,
'station': {'id': 2, 'codes': ['SDZ54421'], 'name': 'Shangdianzi',
'coordinates': {'lat': 40.65, 'lng': 117.106, 'alt': 293.9},
'coordinate_validation_status': 'not checked',
@@ -357,7 +353,6 @@ class TestApps:
'toar1_category': 'unclassified',
'wheat_production_year2000': -999.0},
'changelog': []},
- 'station_id': 2,
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}}
assert response.json() == expected_resp
@@ -526,11 +521,9 @@ class TestApps:
'toar1_category': 'unclassified',
'wheat_production_year2000': -999.0},
'changelog': []},
- 'station_id': 2,
'variable': {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1',
'chemical_formula': 'C7H8', 'id': 7},
- 'variable_id': 7,
'programme': {'id': 0, 'name': '', 'longname': '', 'homepage': '', 'description': ''}
}]
assert response.json() == expected_response
diff --git a/toardb/timeseries/timeseries.py b/toardb/timeseries/timeseries.py
index c821b17816f44950719155e170e75be2dc00b2c6..5da3b33df8aa4bee83b9cc98b38d34bf0200da50 100644
--- a/toardb/timeseries/timeseries.py
+++ b/toardb/timeseries/timeseries.py
@@ -82,26 +82,6 @@ def get_timeseries_programme(name: str, db: Session = Depends(get_db)):
db_programme = crud.get_timeseries_programme(db, name=name)
return db_programme
-@router.get('/check_analysis_params/')
-def check_analysis_params(request: Request):
- # determine allowed query parameters (first *only* from Timeseries)
- timeseries_params = {column.name for column in inspect(Timeseries).c} | {"has_role"}
- gis_params = {"bounding_box", "altitude_range"}
- core_params = {column.name for column in inspect(StationmetaCore).c}
- global_params = {column.name for column in inspect(StationmetaGlobal).c if column.name not in ['id','station_id']}
- data_params = {column.name for column in inspect(Data).c}
- allowed_params = {"limit", "offset"} | gis_params | core_params | global_params | timeseries_params | data_params
-
- status_code=200
- message = "unknown argument(s):"
- for param in request.query_params:
- if param not in allowed_params: #inform user, that an unknown parameter name was used (this could be a typo and falsify the result!)
- message = message + f" {param},"
- status_code=422
- message = message[:-1] + '.'
- if status_code == 200:
- message = ''
- return JSONResponse(status_code=status_code, content=message)
#some more gets to be tested:
#
diff --git a/toardb/toardb.py b/toardb/toardb.py
index 60bbb234d14962dc1b3f2e5f03d89396d16b656d..dc36e4febad696bf9c50d9746abc593a19b5bce9 100644
--- a/toardb/toardb.py
+++ b/toardb/toardb.py
@@ -58,14 +58,14 @@ CT_vocabulary = 0
controlled_fields = 0
app = FastAPI()
-app.mount("/static", StaticFiles(directory="static_fastapi"), name="static_fastapi")
+app.mount("/static", StaticFiles(directory="static/_static"), name="_static")
templates = Jinja2Templates(directory="templates")
@app.middleware("http")
async def response_to_csv(request: Request, call_next):
response = await call_next(request)
if ((response.status_code != 200) or
- (request['path'].startswith(('/data/timeseries/','/ontology/','/controlled_vocabulary/'))) or
+ (request['path'].startswith(('/data/timeseries/','/data/timeseries_with_staging/','/ontology/','/controlled_vocabulary/'))) or
(request.query_params.get('format') != 'csv')):
return response
# from: https://stackoverflow.com/a/71883126
diff --git a/toardb/utils/utils.py b/toardb/utils/utils.py
index 921728eabc70824a03757711c1483fe510556997..3678f44fba3c4bcc22cb03c193151e52e5079b7c 100644
--- a/toardb/utils/utils.py
+++ b/toardb/utils/utils.py
@@ -16,10 +16,11 @@ import requests
import datetime as dt
from toardb.utils.settings import base_geodata_url
-from toardb.contacts.models import Contact, Organisation
+from toardb.contacts.models import Contact, Organisation, Person
from toardb.timeseries.models import Timeseries, TimeseriesRole, timeseries_timeseries_roles_table
from toardb.stationmeta.models import StationmetaCore, StationmetaGlobal
from toardb.data.models import Data
+from toardb.variables.models import Variable
import toardb
# function to return code for given value
@@ -85,10 +86,15 @@ def create_filter(query_params, endpoint):
# determine allowed query parameters (first *only* from Timeseries)
timeseries_params = {column.name for column in inspect(Timeseries).c} | {"has_role"}
+ roles_params = {column.name for column in inspect(TimeseriesRole).c}
gis_params = {"bounding_box", "altitude_range"}
core_params = {column.name for column in inspect(StationmetaCore).c}
global_params = {column.name for column in inspect(StationmetaGlobal).c if column.name not in ['id','station_id']}
data_params = {column.name for column in inspect(Data).c} | {"daterange", "format"}
+ ambig_params = {"station_id", "station_changelog", "station_country"}
+ variable_params = {column.name for column in inspect(Variable).c}
+ person_params = {column.name for column in inspect(Person).c}
+ allrel_params = {"limit", "offset", "fields", "format"}
# pagination
offset= int(query_params.get("offset", 0))
@@ -107,61 +113,41 @@ def create_filter(query_params, endpoint):
raise KeyError(f"An unknown argument was received: fields.")
format = query_params.get("format", 'json')
- allowed_params = {"limit", "offset", "fields", "format"}
+ allowed_params = allrel_params.copy()
if endpoint in {'stationmeta'}:
allowed_params |= gis_params | core_params | global_params
elif endpoint in {'timeseries'}:
- allowed_params |= timeseries_params
+ allowed_params |= timeseries_params | roles_params
elif endpoint in {'search'}:
- allowed_params |= gis_params | core_params | global_params | timeseries_params
+ allowed_params |= gis_params | core_params | global_params | timeseries_params | roles_params | ambig_params
elif endpoint in {'data'}:
allowed_params = {"limit", "offset" } | data_params
- elif endpoint in {'analysis'}:
- allowed_params |= gis_params | core_params | global_params | timeseries_params | data_params
+ elif endpoint in {'variables'}:
+ allowed_params |= variable_params
+ elif endpoint in {'persons'}:
+ allowed_params |= person_params
else:
raise ValueError(f"Wrong endpoint given: {endpoint}")
+ if fields:
+ for field in fields.split(','):
+ if field not in allowed_params:
+ raise ValueError(f"Wrong field given: {field}")
t_filter = []
t_r_filter = []
s_c_filter = []
s_g_filter = []
d_filter = []
+ v_filter = []
+ p_filter = []
# query_params is a multi-dict!
for param in query_params:
if param not in allowed_params: #inform user, that an unknown parameter name was used (this could be a typo and falsify the result!)
raise KeyError(f"An unknown argument was received: {param}.")
- if param in {"limit", "offset"}:
+ if param in allrel_params:
continue
values = [item.strip() for v in query_params.getlist(param) for item in v.split(',')]
- if param in timeseries_params:
- #check for parameters of the controlled vocabulary
- if param == "sampling_frequency":
- values = translate_convoc_list(values, toardb.toardb.SF_vocabulary, "sampling_frequency")
- elif param == "aggregation":
- values = translate_convoc_list(values, toardb.toardb.AT_vocabulary, "aggregation")
- elif param == "data_origin_type":
- values = translate_convoc_list(values, toardb.toardb.OT_vocabulary, "data origin type")
- elif param == "data_origin":
- values = translate_convoc_list(values, toardb.toardb.DO_vocabulary, "data origin")
- if param == "has_role":
- operator = "IN"
- join_operator = "OR"
- if (values[0][0] == '~'):
- operator = "NOT IN"
- join_operator = "AND"
- values[0] = values[0][1:]
- t_r_filter.append(f"organisations.longname {operator} {values}")
- t_r_filter.append(f"organisations.name {operator} {values}")
- t_r_filter.append(f"organisations.city {operator} {values}")
- t_r_filter.append(f"organisations.homepage {operator} {values}")
- t_r_filter.append(f"organisations.contact_url {operator} {values}")
- t_r_filter.append(f"persons.email {operator} {values}")
- t_r_filter.append(f"persons.name {operator} {values}")
- t_r_filter.append(f"persons.orcid {operator} {values}")
- t_r_filter = f" {join_operator} ".join(t_r_filter)
- else:
- t_filter.append(f"timeseries.{param} IN {values}")
- elif param in core_params:
+ if endpoint in ["stationmeta", "timeseries", "search"] and param in core_params:
#check for parameters of the controlled vocabulary
if param == "timezone":
values = translate_convoc_list(values, toardb.toardb.TZ_vocabulary, "timezone")
@@ -184,7 +170,7 @@ def create_filter(query_params, endpoint):
s_c_filter.append(f"LOWER(stationmeta_core.name) LIKE '%{values[0].lower()}%'")
else:
s_c_filter.append(f"stationmeta_core.{param} IN {values}")
- elif param in global_params:
+ elif endpoint in ["stationmeta", "search"] and param in global_params:
if param == "climatic_zone_year2016":
values = translate_convoc_list(values, toardb.toardb.CZ_vocabulary, "climatic zone year2016")
elif param == "toar1_category":
@@ -196,13 +182,41 @@ def create_filter(query_params, endpoint):
elif param == "dominant_ecoregion_year2017":
values = translate_convoc_list(values, toardb.toardb.ER_vocabulary, "ECO region type")
s_g_filter.append(f"stationmeta_global.{param} IN {values}")
- elif param in gis_params:
+ elif endpoint in ["stationmeta", "search"] and param in gis_params:
if param == "bounding_box":
min_lat, min_lon, max_lat, max_lon = values
bbox= f'SRID=4326;POLYGON (({min_lon} {min_lat}, {min_lon} {max_lat}, {max_lon} {max_lat}, {max_lon} {min_lat}, {min_lon} {min_lat}))'
s_c_filter.append(f"ST_CONTAINS(ST_GeomFromEWKT('{bbox}'), coordinates)")
else:
s_c_filter.append(f"ST_Z(coordinates) BETWEEN {values[0]} AND {values[1]}")
+ elif endpoint in ["timeseries", "search"]:
+ #check for parameters of the controlled vocabulary
+ if param == "sampling_frequency":
+ values = translate_convoc_list(values, toardb.toardb.SF_vocabulary, "sampling_frequency")
+ elif param == "aggregation":
+ values = translate_convoc_list(values, toardb.toardb.AT_vocabulary, "aggregation")
+ elif param == "data_origin_type":
+ values = translate_convoc_list(values, toardb.toardb.OT_vocabulary, "data origin type")
+ elif param == "data_origin":
+ values = translate_convoc_list(values, toardb.toardb.DO_vocabulary, "data origin")
+ if param == "has_role":
+ operator = "IN"
+ join_operator = "OR"
+ if (values[0][0] == '~'):
+ operator = "NOT IN"
+ join_operator = "AND"
+ values[0] = values[0][1:]
+ t_r_filter.append(f"organisations.longname {operator} {values}")
+ t_r_filter.append(f"organisations.name {operator} {values}")
+ t_r_filter.append(f"organisations.city {operator} {values}")
+ t_r_filter.append(f"organisations.homepage {operator} {values}")
+ t_r_filter.append(f"organisations.contact_url {operator} {values}")
+ t_r_filter.append(f"persons.email {operator} {values}")
+ t_r_filter.append(f"persons.name {operator} {values}")
+ t_r_filter.append(f"persons.orcid {operator} {values}")
+ t_r_filter = f" {join_operator} ".join(t_r_filter)
+ else:
+ t_filter.append(f"timeseries.{param} IN {values}")
elif param in data_params:
if param == "daterange":
start_date = dt.datetime.fromisoformat(values[0])
@@ -213,6 +227,10 @@ def create_filter(query_params, endpoint):
continue
else:
d_filter.append(f"data.{param} IN {values}")
+ elif endpoint == "variables":
+ v_filter.append(f"variables.{param} IN {values}")
+ elif param in person_params:
+ p_filter.append(f"persons.{param} IN {values}")
t_filter = " AND ".join(t_filter).replace('[','(').replace(']',')')
@@ -222,7 +240,17 @@ def create_filter(query_params, endpoint):
s_c_filter = " AND ".join(s_c_filter).replace('[','(').replace(']',')')
s_g_filter = " AND ".join(s_g_filter).replace('[','(').replace(']',')')
d_filter = " AND ".join(d_filter).replace('[','(').replace(']',')')
- return limit, offset, fields, format, t_filter, t_r_filter, s_c_filter, s_g_filter, d_filter
+ v_filter = " AND ".join(v_filter).replace('[','(').replace(']',')')
+ p_filter = " AND ".join(p_filter).replace('[','(').replace(']',')')
+ filters = {}
+ filters["t_filter"] = t_filter
+ filters["t_r_filter"] = t_r_filter
+ filters["s_c_filter"] = s_c_filter
+ filters["s_g_filter"] = s_g_filter
+ filters["d_filter"] = d_filter
+ filters["v_filter"] = v_filter
+ filters["p_filter"] = p_filter
+ return limit, offset, fields, format, filters
###
diff --git a/toardb/variables/crud.py b/toardb/variables/crud.py
index 4d90790543e47be149ec8f2a698aece2ea3cc964..dfcc59f32bfb6e55bc2e4772d79cdbbd81aae735 100644
--- a/toardb/variables/crud.py
+++ b/toardb/variables/crud.py
@@ -6,8 +6,11 @@ Create, Read, Update, Delete functionality
"""
+from fastapi.responses import JSONResponse
+from sqlalchemy import text
from sqlalchemy.orm import Session
from . import models, schemas
+from toardb.utils.utils import create_filter
def get_variable(db: Session, variable_id: int):
return db.query(models.Variable).filter(models.Variable.id == variable_id).first()
@@ -17,10 +20,28 @@ def get_variable_by_name(db: Session, name: str):
return db.query(models.Variable).filter(models.Variable.name == name.lower()).first()
-def get_variables(db: Session, limit: int, offset: int = 0):
- if limit == "None":
- limit = None
- return db.query(models.Variable).order_by(models.Variable.id).offset(offset).limit(limit).all()
+def get_variables(db: Session, path_params, query_params):
+ try:
+ limit, offset, fields, format, filters = create_filter(query_params, "variables")
+ v_filter = filters["v_filter"]
+ except (KeyError, ValueError) as e:
+ status_code=400
+ return JSONResponse(status_code=status_code, content=str(e))
+
+ if fields:
+ fields2 = fields.split(',')
+ db_objects_l = db.query(*map(text,fields2)).select_from(models.Variable). \
+ filter(text(v_filter)). \
+ order_by(models.Variable.id). \
+ limit(limit).offset(offset)
+ db_objects = []
+ for db_object_immut in db_objects_l:
+ db_object = dict(zip(fields.split(','),db_object_immut))
+ db_objects.append(db_object)
+ else:
+ db_objects = db.query(models.Variable).filter(text(v_filter)). \
+ order_by(models.Variable.id).offset(offset).limit(limit).all()
+ return db_objects
def create_variable(db: Session, variable: schemas.VariableCreate):
diff --git a/toardb/variables/schemas.py b/toardb/variables/schemas.py
index e5a415a069c7b650e6b13368326540b2280ba095..6909cd5fbd346eec464ec6e87b16824bc98b5bd8 100644
--- a/toardb/variables/schemas.py
+++ b/toardb/variables/schemas.py
@@ -12,18 +12,18 @@ from pydantic import BaseModel, Field
class VariableBase(BaseModel):
- name: str = Field(..., description="Name. Short variable-like name of the variable")
- longname: str = Field(..., description="Longname. Long (explicit) name of the variable")
- displayname: str = Field(..., description="Displayname. Display name of the variable")
- cf_standardname: str = Field(..., description="Cf standardname. CF standard name of the variable if defined")
- units: str = Field(..., description="Units. Physical units of variable")
- chemical_formula: str = Field(..., description="Chemical formula. Chemical formula of variable(if applicable)")
+ name: str = Field(None, description="Name. Short variable-like name of the variable")
+ longname: str = Field(None, description="Longname. Long (explicit) name of the variable")
+ displayname: str = Field(None, description="Displayname. Display name of the variable")
+ cf_standardname: str = Field(None, description="Cf standardname. CF standard name of the variable if defined")
+ units: str = Field(None, description="Units. Physical units of variable")
+ chemical_formula: str = Field(None, description="Chemical formula. Chemical formula of variable(if applicable)")
class VariableCreate(VariableBase):
pass
class Variable(VariableBase):
- id: int
+ id: int = None
class Config:
orm_mode = True
diff --git a/toardb/variables/test_variables.py b/toardb/variables/test_variables.py
index df277db850ab65b037eb1a89a2c6f3fd88a4892d..dae70ecd28d8ad326d848c9bf85c38870bee8d70 100644
--- a/toardb/variables/test_variables.py
+++ b/toardb/variables/test_variables.py
@@ -57,8 +57,8 @@ class TestApps:
assert response.json() == expected_resp
- def test_get_all_variables_nolimit(self, client, db):
- response = client.get("/variables/?limit=100")
+ def test_get_all_variables_default_limit(self, client, db):
+ response = client.get("/variables/")
expected_status_code = 200
assert response.status_code == expected_status_code
expected_resp = [{'name': 'benzene', 'longname': 'benzene', 'displayname': 'Benzene',
@@ -86,8 +86,8 @@ class TestApps:
'cf_standardname': 'mole_fraction_of_propane_in_air', 'units': 'nmol mol-1', 'chemical_formula': 'C3H8', 'id': 10}]
assert response.json() == expected_resp
- def test_get_all_variables_nolimit(self, client, db):
- response = client.get("/variables/?limit=100")
+ def test_get_all_variables_unlimited(self, client, db):
+ response = client.get("/variables/?limit=None")
expected_status_code = 200
assert response.status_code == expected_status_code
expected_resp = [{'name': 'benzene', 'longname': 'benzene', 'displayname': 'Benzene',
@@ -162,6 +162,54 @@ class TestApps:
assert response.json() == expected_resp
+ def test_get_all_variables_using_fields(self, client, db):
+ response = client.get("/variables/?limit=None&fields=name,units")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'name': 'benzene', 'units': 'nmol mol-1'},
+ {'name': 'co', 'units': 'nmol mol-1'},
+ {'name': 'no', 'units': 'nmol mol-1'},
+ {'name': 'pm1', 'units': 'µg m-3'},
+ {'name': 'o3', 'units': 'nmol mol-1'},
+ {'name': 'no2', 'units': 'nmol mol-1'},
+ {'name': 'toluene', 'units': 'nmol mol-1'},
+ {'name': 'so2', 'units': 'nmol mol-1'},
+ {'name': 'ethane', 'units': 'nmol mol-1'},
+ {'name': 'propane', 'units': 'nmol mol-1'},
+ {'name': 'ox', 'units': 'nmol mol-1'},
+ {'name': 'aswdir', 'units': 'W/m**2'},
+ {'name': 'pm10', 'units': 'µg m-3'},
+ {'name': 'rn', 'units': 'mBq m-3'},
+ {'name': 'mpxylene', 'units': 'nmol mol-1'},
+ {'name': 'oxylene', 'units': 'nmol mol-1'},
+ {'name': 'ch4', 'units': 'nmol mol-1'},
+ {'name': 'wdir', 'units': 'degrees'},
+ {'name': 'pm2p5', 'units': 'µg m-3'},
+ {'name': 'nox', 'units': 'nmol mol-1'},
+ {'name': 'temp', 'units': 'degC'},
+ {'name': 'wspeed', 'units': 'm s-1'},
+ {'name': 'press', 'units': 'hPa'},
+ {'name': 'cloudcover', 'units': '%'},
+ {'name': 'pblheight', 'units': 'm'},
+ {'name': 'relhum', 'units': '%'},
+ {'name': 'totprecip', 'units': 'kg m-2'},
+ {'name': 'u', 'units': 'm s-1'},
+ {'name': 'v', 'units': 'm s-1'},
+ {'name': 'albedo', 'units': '%'},
+ {'name': 'aswdifu', 'units': 'W/m**2'},
+ {'name': 'humidity', 'units': 'g kg-1'},
+ {'name': 'irradiance', 'units': 'W m-2'}]
+ assert response.json() == expected_resp
+
+
+ def test_get_all_variables_wrong_fieldname(self, client, db):
+ response = client.get("/variables/?limit=None&fields=name,units,bla")
+ expected_status_code = 400
+ assert response.status_code == expected_status_code
+ expected_resp = "Wrong field given: bla"
+ assert response.json() == expected_resp
+
+
def test_get_special(self, client, db):
response = client.get("/variables/id/2")
expected_status_code = 200
@@ -190,6 +238,43 @@ class TestApps:
assert response.json() == expected_resp
+ def test_get_special_by_units(self, client, db):
+ response = client.get("/variables/?units=nmol mol-1")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_resp = [{'name': 'benzene', 'longname': 'benzene', 'displayname': 'Benzene',
+ 'cf_standardname': 'mole_fraction_of_benzene_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'C6H6', 'id': 1},
+ {'name': 'co', 'longname': 'carbon monoxide', 'displayname': 'CO',
+ 'cf_standardname': 'mole_fraction_of_carbon_monoxide_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'CO', 'id': 2},
+ {'name': 'no', 'longname': 'nitrogen monoxide', 'displayname': 'NO',
+ 'cf_standardname': 'mole_fraction_of_nitrogen_monoxide_in_air',
+ 'units': 'nmol mol-1', 'chemical_formula': 'NO', 'id': 3},
+ {'name': 'o3', 'longname': 'ozone', 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3', 'id': 5},
+ {'name': 'no2', 'longname': 'nitrogen dioxide', 'displayname': 'NO2',
+ 'cf_standardname': 'mole_fraction_of_nitrogen_dioxide_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'NO2', 'id': 6},
+ {'name': 'toluene', 'longname': 'toluene', 'displayname': 'Toluene',
+ 'cf_standardname': 'mole_fraction_of_toluene_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'C7H8', 'id': 7},
+ {'name': 'so2', 'longname': 'Sulphur dioxide', 'displayname': 'SO2',
+ 'cf_standardname': 'mole_fraction_of_sulfur_dioxide_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'SO2', 'id': 8},
+ {'name': 'ethane', 'longname': 'Ethane', 'displayname': 'Ethane',
+ 'cf_standardname': 'mole_fraction_of_ethane_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'C2H6', 'id': 9},
+ {'name': 'propane', 'longname': 'Propane', 'displayname': 'Propane',
+ 'cf_standardname': 'mole_fraction_of_propane_in_air', 'units': 'nmol mol-1',
+ 'chemical_formula': 'C3H8', 'id': 10},
+ {'name': 'ox', 'longname': 'Ox', 'displayname': 'Ox',
+ 'cf_standardname': '', 'units': 'nmol mol-1',
+ 'chemical_formula': '', 'id': 11}]
+ assert response.json() == expected_resp
+
+
def test_get_special_by_name_not_existing(self, client, db):
response = client.get("/variables/sabinen")
expected_status_code = 404
diff --git a/toardb/variables/variables.py b/toardb/variables/variables.py
index b97a4b72ad06b8b6e7c3dfacaba02adcd351e3c7..2d387774a19eb53d5e2c85c579535dea71cc70a7 100644
--- a/toardb/variables/variables.py
+++ b/toardb/variables/variables.py
@@ -6,7 +6,7 @@ Simple test API for variable management
"""
from typing import List
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Request
from sqlalchemy.orm import Session
from toardb.variables import crud, schemas
from toardb.utils.database import ToarDbSession, get_db
@@ -21,9 +21,9 @@ def create_variable(variable: schemas.VariableCreate, db: Session = Depends(get_
raise HTTPException(status_code=400, detail="Variable already registered.")
return crud.create_variable(db=db, variable=variable)
-@router.get('/variables/', response_model=List[schemas.Variable])
-def get_variables(offset: int = 0, limit: int|str = 10, db: Session = Depends(get_db)):
- variables = crud.get_variables(db, offset=offset, limit=limit)
+@router.get('/variables/', response_model=List[schemas.Variable], response_model_exclude_none=True, response_model_exclude_unset=True)
+def get_variables(request: Request, db: Session = Depends(get_db)):
+ variables = crud.get_variables(db, path_params=request.path_params, query_params=request.query_params)
return variables
@router.get('/variables/{name}', response_model=schemas.Variable)