#3: toar_controlled_vocabulary is used for downloading timeseries, pytests are...

#3: toar_controlled_vocabulary is used for downloading timeseries, pytests are added for timeseries; not yet working: how to show human readable fields inside nested models (how is this 'conversion' done automatically?)

production_tests.sh

@@ -30,11 +30,8 @@ curl -X POST -H "Content-Type:application/json" -d '{"stationmeta": {"codes":["t
 
 curl http://127.0.0.1:8000/timeseries/
 curl http://127.0.0.1:8000/timeseries/2
-# timeseries upload without nested fields
-curl -X POST -H "Content-Type:application/json" -d '{"timeseries": {"label": "CMA2", "order": 1, "access_rights": 0, "sampling_frequency": 0, "aggregation": 0, "data_start_date": "2003-09-07T15:30:00+02:00", "data_end_date": "2016-12-31T14:30:00+01:00", "measurement_method": "UV absorption", "sampling_height": 7.0, "date_added": "2020-05-15T15:30:00+02:00", "date_modified": "2020-05-16T09:30:00+02:00", "station_id": 2, "variable_id": 7, "additional_metadata":"{}"}}' http://127.0.0.1:8000/timeseries/
-# nested upload with enum fields
-curl -X POST -H "Content-Type:application/json" -d '{"timeseries": {"label": "CMA5", "order": 1, "access_rights": 0, "sampling_frequency": 0, "aggregation": 0, "data_start_date": "2003-09-07T15:30:00+02:00", "data_end_date": "2016-12-31T14:30:00+01:00", "measurement_method": "UV absorption", "sampling_height": 7.0, "date_added": "2020-05-15T15:30:00+02:00", "date_modified": "2020-05-16T09:30:00+02:00", "station_id": 2, "variable_id": 7, "additional_metadata":"{}", "roles": [{"role": 0, "person_id": 3, "status": 0},{"role": 1, "person_id": 3, "status": 0}]}}' http://127.0.0.1:8000/timeseries/
-# TBD: (nested) upload with human readable fields
-curl -X POST -H "Content-Type:application/json" -d '{"timeseries15:30:00+02:00", "data_end_date": "2016-12-31T14:30:00+01:00", "measurement_method": "UV absorption", "sampling_height": 7.0, "date_added": "2020-05-15T15:30:00+02:00", "date_modified": "2020-05-16T09:30:00+02:00", "station_id": 2, "variable_id": 7, "additional_metadata":"{}", "roles": [{"role": "PointOfContact", "person": "s.schroeder@fz-juelich.de", "status": "active"},{"role": "Originator", "person": "Stefan.Feigenspan@uba.de", "status": "active"}]}}' http://127.0.0.1:8000/timeseries/
+# timeseries upload with (nested) human readable fields
+curl -X POST -H "Content-Type:application/json" -d '{"timeseries": {"label": "CMA5", "order": 1, "access_rights": "ByAttribution", "sampling_frequency": "Hourly", "aggregation": "Mean", "source": "Measurement", "data_start_date": "2003-09-07T15:30:00+02:00", "data_end_date": "2016-12-31T14:30:00+01:00", "measurement_method": "UV absorption", "sampling_height": 7.0, "date_added": "2020-05-15T15:30:00+02:00", "date_modified": "2020-05-16T09:30:00+02:00", "station_id": 2, "variable_id": 7, "additional_metadata":"{}", "roles": [{"role": "PointOfContact", "person_id": 3, "status": "active"},{"role": "Originator", "person_id": 1, "status": "active"}]}}' http://127.0.0.1:8000/timeseries/
+# TBD: use person_email (instead of person_id) to identify persons, f. ex. "person_email": "s.schroeder@fz-juelich.de"
 
 curl -X POST -H 'Content-Type: multipart/form-data; charset=utf-8; boundary=__X_PAW_BOUNDARY__' -F "file=@o3_CO002_2012_2017_v1-0.dat" "http://127.0.0.1:8000/data/"

toardb/test_base.py

@@ -61,6 +61,11 @@ def test_db_session():
     # Drop all data after each test
     for tbl in reversed(Base.metadata.sorted_tables):
         _db_conn.execute(tbl.delete())
+    # all tables from "toar_controlled_vocabulary" got lost by the above command!
+    fake_conn = _db_conn.raw_connection()
+    fake_cur = fake_conn.cursor()
+    fake_cur.execute("DROP EXTENSION toar_controlled_vocabulary")
+    fake_conn.commit()
     # put back the connection to the connection pool
     session.close()
 

toardb/timeseries/crud.py

@@ -9,10 +9,11 @@ from sqlalchemy.orm import Session
 from fastapi.responses import JSONResponse
 from . import models
 from .models import timeseries_timeseries_roles_table, \
-                    timeseries_timeseries_annotations_table, timeseries_timeseries_programmes_table
+                    timeseries_timeseries_annotations_table, timeseries_timeseries_programmes_table, \
+                    DA_enum, SF_enum, AT_enum, DS_enum
 from toardb.generic.models import RS_enum, RC_enum
 from .schemas import TimeseriesCreate
-from toardb.utils.utils import get_value_from_str
+from toardb.utils.utils import get_value_from_str, get_str_from_value
 
 
 def get_timeseries(db: Session, timeseries_id: int):
@@ -20,6 +21,10 @@ def get_timeseries(db: Session, timeseries_id: int):
     # there is a mismatch with additional_metadata
     if db_object:
         db_object.additional_metadata = str(db_object.additional_metadata)
+        db_object.access_rights = get_str_from_value(db,DA_enum,db_object.access_rights)
+        db_object.sampling_frequency = get_str_from_value(db,SF_enum,db_object.sampling_frequency)
+        db_object.aggregation = get_str_from_value(db,AT_enum,db_object.aggregation)
+        db_object.source = get_str_from_value(db,DS_enum,db_object.source)
     return db_object
 
 
@@ -28,6 +33,10 @@ def get_all_timeseries(db: Session, skip : int = 0, limit: int = None):
     for db_object in db_objects:
         # there is a mismatch with additional_metadata
         db_object.additional_metadata = str(db_object.additional_metadata)
+        db_object.access_rights = get_str_from_value(db,DA_enum,db_object.access_rights)
+        db_object.sampling_frequency = get_str_from_value(db,SF_enum,db_object.sampling_frequency)
+        db_object.aggregation = get_str_from_value(db,AT_enum,db_object.aggregation)
+        db_object.source = get_str_from_value(db,DS_enum,db_object.source)
     return db_objects
 
 
@@ -39,6 +48,10 @@ def get_timeseries_by_unique_constraints(db: Session, station_id: int, variable_
     # there is a mismatch with additional_metadata
     if db_object:
         db_object.additional_metadata = str(db_object.additional_metadata)
+        db_object.access_rights = get_str_from_value(db,DA_enum,db_object.access_rights)
+        db_object.sampling_frequency = get_str_from_value(db,SF_enum,db_object.sampling_frequency)
+        db_object.aggregation = get_str_from_value(db,AT_enum,db_object.aggregation)
+        db_object.source = get_str_from_value(db,DS_enum,db_object.source)
     return db_object
 
 
@@ -81,6 +94,10 @@ def create_timeseries(db: Session, timeseries: TimeseriesCreate):
     annotations_data = timeseries_dict.pop('annotations', None)
     programmes_data = timeseries_dict.pop('programmes', None)
     db_timeseries = models.Timeseries(**timeseries_dict)
+    db_timeseries.access_rights = get_value_from_str(db,DA_enum,db_timeseries.access_rights)
+    db_timeseries.sampling_frequency = get_value_from_str(db,SF_enum,db_timeseries.sampling_frequency)
+    db_timeseries.aggregation = get_value_from_str(db,AT_enum,db_timeseries.aggregation)
+    db_timeseries.source = get_value_from_str(db,DS_enum,db_timeseries.source)
     db.add(db_timeseries)
     result = db.commit()
     db.refresh(db_timeseries)
@@ -137,4 +154,8 @@ def create_timeseries(db: Session, timeseries: TimeseriesCreate):
     # in upload command, we have now: "additional_metadata": "{}"
     # but return from this method gives (=database): "additional_metadata": {}
     db_timeseries.additional_metadata = str(db_timeseries.additional_metadata)
+    db_timeseries.access_rights = get_str_from_value(db,DA_enum,db_timeseries.access_rights)
+    db_timeseries.sampling_frequency = get_str_from_value(db,SF_enum,db_timeseries.sampling_frequency)
+    db_timeseries.aggregation = get_str_from_value(db,AT_enum,db_timeseries.aggregation)
+    db_timeseries.source = get_str_from_value(db,DS_enum,db_timeseries.source)
     return db_timeseries

toardb/timeseries/models_core.py

@@ -31,6 +31,8 @@ class Timeseries(Base):
     +---------------------+--------------------------+-----------+----------+----------------------------------------+
     | aggregation         | integer                  |           | not null |                                        |
     +---------------------+--------------------------+-----------+----------+----------------------------------------+
+    | source              | integer                  |           | not null | 1                                      |
+    +---------------------+--------------------------+-----------+----------+----------------------------------------+
     | data_start_date     | timestamp with time zone |           | not null |                                        |
     +---------------------+--------------------------+-----------+----------+----------------------------------------+
     | data_end_date       | timestamp with time zone |           | not null |                                        |
@@ -84,6 +86,7 @@ class Timeseries(Base):
     access_rights = Column(Integer, nullable=False)
     sampling_frequency = Column(Integer, nullable=False)
     aggregation = Column(Integer, nullable=False)
+    source = Column(Integer, nullable=False, server_default=text("1"))
     data_start_date = Column(DateTime(True), nullable=False)
     data_end_date = Column(DateTime(True), nullable=False)
     measurement_method = Column(String(128), nullable=False)

toardb/timeseries/schemas.py

@@ -15,9 +15,10 @@ class TimeseriesCoreBase(BaseModel):
     id: int = None
     label: str
     order: int
-    access_rights: int
-    sampling_frequency: int
-    aggregation: int
+    access_rights: str
+    sampling_frequency: str
+    aggregation: str
+    source: str
     data_start_date: dt.datetime
     data_end_date: dt.datetime
     measurement_method: str

toardb/timeseries/test_timeseries.py

@@ -28,10 +28,13 @@ class TestApps:
     @pytest.fixture(autouse=True)
     def setup_db_data(self, db):
         _db_conn = get_test_engine()
-        print(_db_conn)
         # id_seq will not be reset automatically between tests!
         fake_conn = _db_conn.raw_connection()
         fake_cur = fake_conn.cursor()
+        # all tables of "toar_controlled_vocabulary get lost inbetween tests
+        # ==> create the extension again!
+        fake_cur.execute("CREATE EXTENSION IF NOT EXISTS toar_controlled_vocabulary")
+        fake_conn.commit()
         fake_cur.execute("ALTER SEQUENCE auth_user_id_seq RESTART WITH 1")
         fake_conn.commit()
         fake_cur.execute("ALTER SEQUENCE variables_id_seq RESTART WITH 1")
@@ -121,13 +124,11 @@ class TestApps:
 
 
     def test_get_timeseries(self, client, db):
-        print("==============Client: ================", client.base_url)
         response = client.get("/timeseries/")
         expected_status_code = 200
         assert response.status_code == expected_status_code
-        print("I got: ", response.json())
-        expected_resp = [{'id': 1, 'label': 'CMA', 'order': 1, 'access_rights': 0,
-                          'sampling_frequency': 0, 'aggregation': 0,
+        expected_resp = [{'id': 1, 'label': 'CMA', 'order': 1, 'access_rights': 'ByAttribution',
+                          'sampling_frequency': 'Hourly', 'aggregation': 'Mean', 'source': 'Measurement',
                           'data_start_date': '2003-09-07T15:30:00+02:00', 'data_end_date': '2016-12-31T14:30:00+01:00',
                           'measurement_method': 'UV absorption', 'sampling_height': 7.0,
                           'date_added': '2020-05-15T15:30:00+02:00', 'date_modified': '2020-05-16T09:30:00+02:00',
@@ -148,8 +149,8 @@ class TestApps:
         response = client.get("/timeseries/1")
         expected_status_code = 200
         assert response.status_code == expected_status_code
-        expected_resp = {'id': 1, 'label': 'CMA', 'order': 1, 'access_rights': 0,
-                         'sampling_frequency': 0, 'aggregation': 0,
+        expected_resp = {'id': 1, 'label': 'CMA', 'order': 1, 'access_rights': 'ByAttribution',
+                         'sampling_frequency': 'Hourly', 'aggregation': 'Mean', 'source': 'Measurement',
                          'data_start_date': '2003-09-07T15:30:00+02:00', 'data_end_date': '2016-12-31T14:30:00+01:00',
                          'measurement_method': 'UV absorption', 'sampling_height': 7.0,
                          'date_added': '2020-05-15T15:30:00+02:00', 'date_modified': '2020-05-16T09:30:00+02:00',
@@ -178,8 +179,8 @@ class TestApps:
     def test_insert_new(self, client, db):
         response = client.post("/timeseries/",
                 json={"timeseries":
-                          {"label": "CMA2", "order": 1, "access_rights": 0,
-                           "sampling_frequency": 0, "aggregation": 0,
+                          {"label": "CMA2", "order": 1, "access_rights": "ByAttribution",
+                           "sampling_frequency": "Hourly", "aggregation": "Mean", "source": "Measurement",
                            "data_start_date": "2003-09-07T15:30:00+02:00",
                            "data_end_date": "2016-12-31T14:30:00+01:00",
                            "measurement_method": "UV absorption", "sampling_height": 7.0,
@@ -190,8 +191,8 @@ class TestApps:
                    )
         expected_status_code = 200
         assert response.status_code == expected_status_code
-        expected_resp = {'id': 2, 'label': 'CMA2', 'order': 1, 'access_rights': 0,
-                         'sampling_frequency': 0, 'aggregation': 0,
+        expected_resp = {'id': 2, 'label': 'CMA2', 'order': 1, 'access_rights': 'ByAttribution',
+                         'sampling_frequency': 'Hourly', 'aggregation': 'Mean', 'source': 'Measurement',
                          'data_start_date': '2003-09-07T15:30:00+02:00', 'data_end_date': '2016-12-31T14:30:00+01:00',
                          'measurement_method': 'UV absorption', 'sampling_height': 7.0,
                          'date_added': '2020-05-15T15:30:00+02:00', 'date_modified': '2020-05-16T09:30:00+02:00',
@@ -203,27 +204,29 @@ class TestApps:
     def test_insert_new_with_roles(self, client, db):
         response = client.post("/timeseries/",
                 json={"timeseries":
-                          {"label": "CMA2", "order": 1, "access_rights": 0,
-                           "sampling_frequency": 0, "aggregation": 0,
+                          {"label": "CMA2", "order": 1, "access_rights": "ByAttribution",
+                           "sampling_frequency": "Hourly", "aggregation": "Mean", "source": "Measurement",
                            "data_start_date": "2003-09-07T15:30:00+02:00",
                            "data_end_date": "2016-12-31T14:30:00+01:00",
                            "measurement_method": "UV absorption", "sampling_height": 7.0,
                            "date_added": "2020-05-15T15:30:00+02:00", "date_modified": "2020-05-16T09:30:00+02:00",
                            "station_id": 2, "variable_id": 7,
                            "additional_metadata":"{}",
-                           "roles": [{"role": 0, "person_id": 3, "status": 0},{"role": 1, "person_id": 3, "status": 0}]}
+                           "roles": [{"role": "PointOfContact", "person_id": 3, "status": "active"},
+                                     {"role": "Originator", "person_id": 1, "status": "active"}]
+                          }
                      }
                    )
         expected_status_code = 200
         assert response.status_code == expected_status_code
-        expected_resp = {'id': 2, 'label': 'CMA2', 'order': 1, 'access_rights': 0,
-                         'sampling_frequency': 0, 'aggregation': 0,
+        expected_resp = {'id': 2, 'label': 'CMA2', 'order': 1, 'access_rights': 'ByAttribution',
+                         'sampling_frequency': 'Hourly', 'aggregation': 'Mean', 'source': 'Measurement',
                          'data_start_date': '2003-09-07T15:30:00+02:00', 'data_end_date': '2016-12-31T14:30:00+01:00',
                          'measurement_method': 'UV absorption', 'sampling_height': 7.0,
                          'date_added': '2020-05-15T15:30:00+02:00', 'date_modified': '2020-05-16T09:30:00+02:00',
                          'station_id': 2, 'variable_id': 7,
                          'additional_metadata':{},
-                         'roles': [{'id': 1, 'person_id': 3, 'role': 0, 'status': 0}, {'id': 2, 'person_id': 3, 'role': 1, 'status': 0}],
+                         'roles': [{'id': 1, 'person_id': 3, 'role': '0', 'status': '0'}, {'id': 2, 'person_id': 1, 'role': '2', 'status': '0'}],
                          'annotations': [], 'programmes': []}
         assert response.json() == expected_resp