Skip to content
Snippets Groups Projects
Commit c2025b8c authored by Sabine Schröder's avatar Sabine Schröder
Browse files

almost done: using human readable controlled vocabulary for upload; TBD: using...

almost done: using human readable controlled vocabulary for upload; TBD: using human readable controlled vocabulary for download, adapting pytests
parent 4612d17d
No related branches found
No related tags found
No related merge requests found
Pipeline #39362 passed
...@@ -11,7 +11,7 @@ curl http://127.0.0.1:8000/contacts/organisations/ ...@@ -11,7 +11,7 @@ curl http://127.0.0.1:8000/contacts/organisations/
curl http://127.0.0.1:8000/contacts/organisations/id/2 curl http://127.0.0.1:8000/contacts/organisations/id/2
curl http://127.0.0.1:8000/contacts/organisations/id/99 curl http://127.0.0.1:8000/contacts/organisations/id/99
curl http://127.0.0.1:8000/contacts/organisations/FZJ curl http://127.0.0.1:8000/contacts/organisations/FZJ
curl -X POST -H "Content-Type:application/json" -d '{"organisation": {"name": "FZJ2", "longname": "Forschungszentrum Test", "kind": 1, "city": "Jülich", "postcode": "52425", "street_address": "Wilhelm-Johnen-Straße", "country": "Germany", "homepage": "https://www.fz-juelich.de"}}' http://127.0.0.1:8000/contacts/organisations/ curl -X POST -H "Content-Type:application/json" -d '{"organisation": {"name": "FZJ2", "longname": "Forschungszentrum Test", "kind": "Research", "city": "Jülich", "postcode": "52425", "street_address": "Wilhelm-Johnen-Straße", "country": "Germany", "homepage": "https://www.fz-juelich.de"}}' http://127.0.0.1:8000/contacts/organisations/
curl http://127.0.0.1:8000/contacts/persons/ curl http://127.0.0.1:8000/contacts/persons/
curl http://127.0.0.1:8000/contacts/persons/id/3 curl http://127.0.0.1:8000/contacts/persons/id/3
curl http://127.0.0.1:8000/contacts/persons/id/99 curl http://127.0.0.1:8000/contacts/persons/id/99
...@@ -25,12 +25,8 @@ curl http://127.0.0.1:8000/stationmeta_core/China11 ...@@ -25,12 +25,8 @@ curl http://127.0.0.1:8000/stationmeta_core/China11
curl http://127.0.0.1:8000/stationmeta/China11 curl http://127.0.0.1:8000/stationmeta/China11
# station upload without nested fields # station upload without nested fields
curl -X POST -H "Content-Type:application/json" -d '{"stationmeta": {"codes":["ttt3","ttt4"],"name":"Test_China","coordinates":{"lat":36.256,"lng":17.106,"alt":1534.0},"country":"China","state":"Shandong Sheng","coordinate_validation_status":0,"coordinate_validation_date":"2020-03-11T12:22:18.047974+01:00","type_of_environment":0,"type_of_area":0,"category":"","timezone":"", "coordinate_validator_id": 1, "additional_metadata":"{}"}}' "http://127.0.0.1:8000/stationmeta/" curl -X POST -H "Content-Type:application/json" -d '{"stationmeta": {"codes":["ttt3","ttt4"],"name":"Test_China","coordinates":{"lat":36.256,"lng":17.106,"alt":1534.0},"country":"China","state":"Shandong Sheng","coordinate_validation_status":0,"coordinate_validation_date":"2020-03-11T12:22:18.047974+01:00","type_of_environment":0,"type_of_area":0,"category":"","timezone":"", "coordinate_validator_id": 1, "additional_metadata":"{}"}}' "http://127.0.0.1:8000/stationmeta/"
# nested upload with enum fields # (nested) upload including 'global' (not defining all metadata -- getting the rest from default values) with human readable fields
curl -X POST -H "Content-Type:application/json" -d '{"stationmeta": {"codes":["ttt3","ttt4"],"name":"Test_China","coordinates":{"lat":36.256,"lng":17.106,"alt":1534.0},"country":"China","state":"Shandong Sheng","coordinate_validation_status":0,"coordinate_validation_date":"2020-03-11T12:22:18.047974+01:00","type_of_environment":0,"type_of_area":0,"category":"","timezone":"", "coordinate_validator_id": 1, "additional_metadata":"{}", roles": [{"role": 0, "person_id": 3, "status": 0},{"role": 1, "person_id": 3, "status": 0}]}}' "http://127.0.0.1:8000/stationmeta/" curl -X POST -H "Content-Type:application/json" -d '{"stationmeta": {"codes":["ttt71"],"name":"Ttt_China71","coordinates":{"lat":36.256,"lng":17.106,"alt":1534.0},"country":"China","state":"Shandong Sheng","coordinate_validation_status":0,"coordinate_validation_date":"2020-03-11T12:22:18.047974+01:00","type_of_environment":0,"type_of_area":0,"category":"","timezone":"", "coordinate_validator_id": 1, "additional_metadata":"{}", "roles": [{"role": 0, "person_id": 3, "status": 0},{"role": 1, "person_id": 3, "status": 0}], "globalmeta": {"climatic_zone": "WarmTemperateMoist"}}}' http://127.0.0.1:8000/stationmeta/
# nested upload including 'global' (not defining all metadata -- getting the rest from default values)
curl -X POST -H "Content-Type:application/json" -d '{"stationmeta": {"codes":["ttt58"],"name":"Ttt_China58","coordiShandong Sheng","coordinate_validation_status":0,"coordinate_validation_date":"2020-03-11T12:22:18.047974+01:00","type_of_environment":0,"type_of_area":0,"category":"","timezone":"", "coordinate_validator_id": 1, "additional_metadata":"{}", "roles": [{"role": 0, "person_id": 3, "status": 0},{"role": 1, "person_id": 3, "status": 0}], "globalmeta": {"climatic_zone":1}}}' http://127.0.0.1:8000/stationmeta/
# TBD: (nested) upload with human readable fields
curl -X POST -H "Content-Type:application/json" -d '{"stationmeta": {"codes":["ttt3","ttt4"],"name":"Test_China","coordinates":{"lat":36.256,"lng":17.106,"alt":1534.0},"country":"China","state":"Shandong Sheng","coordinate_validation_status":0,"coordinate_validation_date":"2020-03-11T12:22:18.047974+01:00","type_of_environment":0,"type_of_area":0,"category":"","timezone":"", "coordinate_validator_id": 1, "additional_metadata":"{}", "roles": [{"role": "PointOfContact", "person": "s.schroeder@fz-juelich.de", "status": "active"},{"role": "Originator", "person": "Stefan.Feigenspan@uba.de", "status": "active"}]}}' "http://127.0.0.1:8000/stationmeta/"
curl http://127.0.0.1:8000/timeseries/ curl http://127.0.0.1:8000/timeseries/
curl http://127.0.0.1:8000/timeseries/2 curl http://127.0.0.1:8000/timeseries/2
......
...@@ -7,7 +7,9 @@ Create, Read, Update, Delete functionality ...@@ -7,7 +7,9 @@ Create, Read, Update, Delete functionality
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from . import models from . import models
from .models import OK_enum
from .schemas import PersonCreate, OrganisationCreate from .schemas import PersonCreate, OrganisationCreate
from toardb.utils.utils import get_value_from_str
def get_organisation(db: Session, organisation_id: int): def get_organisation(db: Session, organisation_id: int):
...@@ -23,6 +25,7 @@ def get_organisation_by_name(db: Session, name: str): ...@@ -23,6 +25,7 @@ def get_organisation_by_name(db: Session, name: str):
def create_organisation(db: Session, organisation: OrganisationCreate): def create_organisation(db: Session, organisation: OrganisationCreate):
db_organisation = models.Organisation(**organisation.dict()) db_organisation = models.Organisation(**organisation.dict())
db_organisation.kind = get_value_from_str(db,OK_enum,db_organisation.kind)
db.add(db_organisation) db.add(db_organisation)
result = db.commit() result = db.commit()
db.refresh(db_organisation) db.refresh(db_organisation)
......
...@@ -4,3 +4,16 @@ from toardb.base import Base ...@@ -4,3 +4,16 @@ from toardb.base import Base
from .models_person import Person from .models_person import Person
from .models_organisation import Organisation from .models_organisation import Organisation
from sqlalchemy import Table, Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
# controlled vocabulary
# Kind of Organizations
OK_enum = Table("ok_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
...@@ -11,7 +11,7 @@ class OrganisationBase(BaseModel): ...@@ -11,7 +11,7 @@ class OrganisationBase(BaseModel):
id: int = None id: int = None
name: str name: str
longname: str longname: str
kind: int kind: str
city: str city: str
postcode: str postcode: str
street_address: str street_address: str
......
from sqlalchemy import Table, Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from toardb.base import Base
# controlled vocabulary
# Role Status
RS_enum = Table("rs_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
# Role Codes
RC_enum = Table("rc_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
...@@ -13,9 +13,12 @@ from fastapi import File, UploadFile ...@@ -13,9 +13,12 @@ from fastapi import File, UploadFile
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from fastapi.encoders import jsonable_encoder from fastapi.encoders import jsonable_encoder
from . import models from . import models
from .models import stationmeta_core_stationmeta_roles_table, stationmeta_core_stationmeta_annotations_table from .models import stationmeta_core_stationmeta_roles_table, stationmeta_core_stationmeta_annotations_table, \
CZ_enum
from toardb.generic.models import RS_enum, RC_enum
from .schemas import get_coordinates_from_geom, get_geom_from_coordinates, StationmetaCreate, Coordinates from .schemas import get_coordinates_from_geom, get_geom_from_coordinates, StationmetaCreate, Coordinates
from pydantic import ValidationError from pydantic import ValidationError
from toardb.utils.utils import get_value_from_str
def get_stationmeta_core(db: Session, station_code: str): def get_stationmeta_core(db: Session, station_code: str):
...@@ -96,9 +99,10 @@ def create_stationmeta(db: Session, stationmeta: StationmetaCreate): ...@@ -96,9 +99,10 @@ def create_stationmeta(db: Session, stationmeta: StationmetaCreate):
stationmeta_core_id = db_stationmeta.id stationmeta_core_id = db_stationmeta.id
# store roles and update association table # store roles and update association table
if roles_data: if roles_data:
print("====roles_data============", roles_data)
for r in roles_data: for r in roles_data:
db_role = models.StationmetaRole(**r) db_role = models.StationmetaRole(**r)
db_role.role = get_value_from_str(db,RC_enum,db_role.role)
db_role.status = get_value_from_str(db,RS_enum,db_role.status)
# check whether role is already present in database # check whether role is already present in database
db_object = get_unique_stationmeta_role(db, db_role.role, db_role.person_id, db_role.status) db_object = get_unique_stationmeta_role(db, db_role.role, db_role.person_id, db_role.status)
if db_object: if db_object:
...@@ -152,6 +156,8 @@ def create_stationmeta(db: Session, stationmeta: StationmetaCreate): ...@@ -152,6 +156,8 @@ def create_stationmeta(db: Session, stationmeta: StationmetaCreate):
# store globalmeta # store globalmeta
if globalmeta_data: if globalmeta_data:
db_global = models.StationmetaGlobal(**globalmeta_data) db_global = models.StationmetaGlobal(**globalmeta_data)
if db_global.climatic_zone:
db_global.climatic_zone = get_value_from_str(db,CZ_enum,db_global.climatic_zone)
db_global.station_id = stationmeta_core_id db_global.station_id = stationmeta_core_id
db.add(db_global) db.add(db_global)
db.commit() db.commit()
......
...@@ -27,3 +27,51 @@ CV_enum = Table("cv_vocabulary", ...@@ -27,3 +27,51 @@ CV_enum = Table("cv_vocabulary",
Column("enum_display_str", String) Column("enum_display_str", String)
) )
# Station Types
ST_enum = Table("st_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
# Station Types of Area
TA_enum = Table("ta_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
# Station TOAR Categories
TC_enum = Table("tc_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
# Station HTAP Regions (TIER1)
TR_enum = Table("tr_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
# Station Dominant Landcover Types
DL_enum = Table("dl_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
# Result Types
RT_enum = Table("rt_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
...@@ -151,7 +151,7 @@ class StationmetaGlobalBase(BaseModel): ...@@ -151,7 +151,7 @@ class StationmetaGlobalBase(BaseModel):
id: int = None id: int = None
population_density_year2010: float population_density_year2010: float
max_population_density_25km_year2010: float max_population_density_25km_year2010: float
climatic_zone: int climatic_zone: str
nightlight_1km_year2013: float nightlight_1km_year2013: float
nightlight_5km_year2013: float nightlight_5km_year2013: float
max_nightlight_25km_year2013: float max_nightlight_25km_year2013: float
...@@ -183,7 +183,7 @@ class StationmetaGlobalBaseNested(BaseModel): ...@@ -183,7 +183,7 @@ class StationmetaGlobalBaseNested(BaseModel):
id: int = None id: int = None
population_density_year2010: float = None population_density_year2010: float = None
max_population_density_25km_year2010: float = None max_population_density_25km_year2010: float = None
climatic_zone: int = None climatic_zone: str = None
nightlight_1km_year2013: float = None nightlight_1km_year2013: float = None
nightlight_5km_year2013: float = None nightlight_5km_year2013: float = None
max_nightlight_25km_year2013: float = None max_nightlight_25km_year2013: float = None
......
...@@ -8,9 +8,11 @@ from sqlalchemy import insert ...@@ -8,9 +8,11 @@ from sqlalchemy import insert
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from fastapi.responses import JSONResponse from fastapi.responses import JSONResponse
from . import models from . import models
from .models import RS_enum, RC_enum, timeseries_timeseries_roles_table, \ from .models import timeseries_timeseries_roles_table, \
timeseries_timeseries_annotations_table, timeseries_timeseries_programmes_table timeseries_timeseries_annotations_table, timeseries_timeseries_programmes_table
from toardb.generic.models import RS_enum, RC_enum
from .schemas import TimeseriesCreate from .schemas import TimeseriesCreate
from toardb.utils.utils import get_value_from_str
def get_timeseries(db: Session, timeseries_id: int): def get_timeseries(db: Session, timeseries_id: int):
...@@ -73,17 +75,7 @@ def get_unique_timeseries_annotation(db: Session, text: str, contributor_id: int ...@@ -73,17 +75,7 @@ def get_unique_timeseries_annotation(db: Session, text: str, contributor_id: int
return db_object return db_object
def __get_status_enum(db: Session):
return db.query(RS_enum).all()
def __get_role_enum(db: Session):
return db.query(RC_enum).all()
def create_timeseries(db: Session, timeseries: TimeseriesCreate): def create_timeseries(db: Session, timeseries: TimeseriesCreate):
enum_RS = __get_status_enum(db)
enum_RC = __get_role_enum(db)
timeseries_dict = timeseries.dict() timeseries_dict = timeseries.dict()
roles_data = timeseries_dict.pop('roles', None) roles_data = timeseries_dict.pop('roles', None)
annotations_data = timeseries_dict.pop('annotations', None) annotations_data = timeseries_dict.pop('annotations', None)
...@@ -98,6 +90,8 @@ def create_timeseries(db: Session, timeseries: TimeseriesCreate): ...@@ -98,6 +90,8 @@ def create_timeseries(db: Session, timeseries: TimeseriesCreate):
if roles_data: if roles_data:
for r in roles_data: for r in roles_data:
db_role = models.TimeseriesRole(**r) db_role = models.TimeseriesRole(**r)
db_role.role = get_value_from_str(db,RC_enum,db_role.role)
db_role.status = get_value_from_str(db,RS_enum,db_role.status)
# check whether role is already present in database # check whether role is already present in database
db_object = get_unique_timeseries_role(db, db_role.role, db_role.person_id, db_role.status) db_object = get_unique_timeseries_role(db, db_role.role, db_role.person_id, db_role.status)
if db_object: if db_object:
......
...@@ -4,17 +4,37 @@ from .models_annotation import TimeseriesAnnotation, timeseries_timeseries_annot ...@@ -4,17 +4,37 @@ from .models_annotation import TimeseriesAnnotation, timeseries_timeseries_annot
from .models_programme import TimeseriesProgramme, timeseries_timeseries_programmes_table from .models_programme import TimeseriesProgramme, timeseries_timeseries_programmes_table
from toardb.base import Base from toardb.base import Base
from sqlalchemy import Table, String, Column, ForeignKey, Integer from sqlalchemy import Table, Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
# controlled vocabulary # controlled vocabulary
RS_enum = Table("rs_vocabulary",
# Data Access Rights
DA_enum = Table("da_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
# Sampling Frequencies
SF_enum = Table("sf_vocabulary",
Base.metadata,
Column("enum_val", Integer, primary_key=True),
Column("enum_str", String),
Column("enum_display_str", String)
)
# Aggregation Types
AT_enum = Table("at_vocabulary",
Base.metadata, Base.metadata,
Column("enum_val", Integer, primary_key=True), Column("enum_val", Integer, primary_key=True),
Column("enum_str", String), Column("enum_str", String),
Column("enum_display_str", String) Column("enum_display_str", String)
) )
RC_enum = Table("rc_vocabulary", # Data Sources
DS_enum = Table("ds_vocabulary",
Base.metadata, Base.metadata,
Column("enum_val", Integer, primary_key=True), Column("enum_val", Integer, primary_key=True),
Column("enum_str", String), Column("enum_str", String),
......
...@@ -43,8 +43,8 @@ class TimeseriesCore(TimeseriesCoreBase): ...@@ -43,8 +43,8 @@ class TimeseriesCore(TimeseriesCoreBase):
class TimeseriesRoleBase(BaseModel): class TimeseriesRoleBase(BaseModel):
id: int = None id: int = None
role: int role: str
status: int status: str
person_id: int person_id: int
class TimeseriesRoleCreate(TimeseriesRoleBase): class TimeseriesRoleCreate(TimeseriesRoleBase):
......
# -*- coding: utf-8 -*-
"""
Helper functions for TOAR database
"""
from sqlalchemy import Table
from sqlalchemy.orm import Session
from collections import namedtuple
# get the controlled vocabulary from table
def __get_enum_dict(db: Session, table: Table):
Enumdict=namedtuple("Dict",["value","string","display_str"])
enum_dict = []
for entry in db.query(table).all():
enum_dict.append(Enumdict(*entry))
return enum_dict
# function to return code for given value
def get_str_from_value(db: Session, table: Table, value: int) -> str:
enum_error = "Invalid " + table.name
enum_dict = __get_enum_dict(db, table)
enum_entry = tuple(filter(lambda x: x.value == value, enum_dict))
return enum_entry[0].string
raise ValueError(enum_error)
# function to return value for given code
def get_value_from_str(db: Session, table: Table, string: str) -> int:
enum_error = "Invalid " + table.name
enum_dict = __get_enum_dict(db, table)
enum_entry = tuple(filter(lambda x: x.string == string, enum_dict))
return enum_entry[0].value
raise ValueError(enum_error)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment