#1: added another test (timeseries not existing); tried to have more...

#1: added another test (timeseries not existing); tried to have more user-friendly error messages; still struggling why I get doubled quotation marks now

toardb/data/crud.py

@@ -54,32 +54,37 @@ def create_data(db: Session, engine: Engine, input_handle: UploadFile = File(...
             timeoffset = dt.timedelta(hours=float(line.split(':')[1]))
         prev, pos = pos, f.tell()
     f.seek(prev)
-    stationmeta_core = get_stationmeta_core(db=db,station_code=station_id)
+    station_code = station_id
+    stationmeta_core = get_stationmeta_core(db=db,station_code=station_code)
     station_id = stationmeta_core.id
     timeseries = get_timeseries_by_unique_constraints(db=db,station_id=station_id,variable_id=variable_id,label=label)
-    timeseries_id = timeseries.id
-    # open SpooledTemporaryFile, skip header (and also try to insert timeseries_id!)
-    df = pd.read_csv(input_handle.file, comment='#', header=None, sep=';',names=["time","value","flags"],parse_dates=["time"],index_col="time")
-    # substract timeshift to convert data to UTC
-    df.index = df.index - timeoffset
-    # now insert the timeseries_id to the end of the data frame
-    df.insert(2, 'timeseries_id', timeseries_id)
-    # datetime needs timezone information
-    df = df.tz_localize('UTC')
-    buf = StringIO()
-    df.to_csv(buf, header=False)
-    buf.pos = 0
-    buf.seek(0)
-    fake_conn = engine.raw_connection()
-    fake_cur = fake_conn.cursor()
-    try:
-        fake_cur.copy_from(buf, 'data', sep=',', columns=('datetime','value','flags','timeseries_id'))
-        fake_conn.commit()
-        message = "Data successfully inserted."
-        status_code = 200
-    except:
-        e = sys.exc_info()[0]
-        message = "An error occurred in data insertion: %s" % (e,)
+    if timeseries:
+        timeseries_id = timeseries.id
+        # open SpooledTemporaryFile, skip header (and also try to insert timeseries_id!)
+        df = pd.read_csv(input_handle.file, comment='#', header=None, sep=';',names=["time","value","flags"],parse_dates=["time"],index_col="time")
+        # substract timeshift to convert data to UTC
+        df.index = df.index - timeoffset
+        # now insert the timeseries_id to the end of the data frame
+        df.insert(2, 'timeseries_id', timeseries_id)
+        # datetime needs timezone information
+        df = df.tz_localize('UTC')
+        buf = StringIO()
+        df.to_csv(buf, header=False)
+        buf.pos = 0
+        buf.seek(0)
+        fake_conn = engine.raw_connection()
+        fake_cur = fake_conn.cursor()
+        try:
+            fake_cur.copy_from(buf, 'data', sep=',', columns=('datetime','value','flags','timeseries_id'))
+            fake_conn.commit()
+            message = 'Data successfully inserted.'
+            status_code = 200
+        except:
+            e = sys.exc_info()[0]
+            message = 'An error occurred in data insertion: %s' % (e,)
+            status_code = 400
+    else:
+        message = f'Timeseries not found for station {station_code.strip()}, variable {variable_name}, label {label.strip()}'
         status_code = 400
     return JSONResponse(status_code=status_code, content=message)
 

toardb/data/data.py

@@ -60,9 +60,12 @@ async def create_data(file: UploadFile = File(...), db: Session = Depends(get_db
 #   we want to upload a whole file!
 #
     response = crud.create_data(db, engine, input_handle=file)
-    msg = response.body.decode('utf-8')
-    msg2 = '"An error occurred in data insertion: <class \'psycopg2.errors.UniqueViolation\'>"'
-    if (msg == msg2):
-        raise HTTPException(status_code=400, detail="Data for timeseries already registered.")
+    if response.status_code != 200:
+        msg = response.body.decode('utf-8')
+        # try to parse error messages from DBS (to be more understandable)
+        msg2 = '"An error occurred in data insertion: <class \'psycopg2.errors.UniqueViolation\'>"'
+        if (msg == msg2):
+            msg = 'Data for timeseries already registered.'
+        raise HTTPException(status_code=400, detail=msg)
     return response
 

toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat

+toluene_SDZ54421_2012_2012_v1-0.dat
\ No newline at end of file

toardb/data/test_data.py

@@ -185,3 +185,12 @@ class TestApps:
         assert response.status_code == expected_status_code
         expected_resp = {'detail': 'Data for timeseries already registered.'}
         assert response.json() == expected_resp
+
+                                                    
+    def test_insert_missing_id(self, client, db):
+        response = client.post("/data/", files={"file": open("toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat", "rb")})
+        expected_status_code = 400
+        assert response.status_code == expected_status_code
+        print (response.json())
+        expected_resp = {'detail': '"Timeseries not found for station SDZ54421, variable o3, label CMA"'}
+        assert response.json() == expected_resp