diff --git a/toardb/data/crud.py b/toardb/data/crud.py index b7a2d689d8b8b2e1473e7618f48e130cf2d36243..461cc462806f2b4b440ef964d7579ce13830d4a3 100644 --- a/toardb/data/crud.py +++ b/toardb/data/crud.py @@ -54,32 +54,37 @@ def create_data(db: Session, engine: Engine, input_handle: UploadFile = File(... timeoffset = dt.timedelta(hours=float(line.split(':')[1])) prev, pos = pos, f.tell() f.seek(prev) - stationmeta_core = get_stationmeta_core(db=db,station_code=station_id) + station_code = station_id + stationmeta_core = get_stationmeta_core(db=db,station_code=station_code) station_id = stationmeta_core.id timeseries = get_timeseries_by_unique_constraints(db=db,station_id=station_id,variable_id=variable_id,label=label) - timeseries_id = timeseries.id - # open SpooledTemporaryFile, skip header (and also try to insert timeseries_id!) - df = pd.read_csv(input_handle.file, comment='#', header=None, sep=';',names=["time","value","flags"],parse_dates=["time"],index_col="time") - # substract timeshift to convert data to UTC - df.index = df.index - timeoffset - # now insert the timeseries_id to the end of the data frame - df.insert(2, 'timeseries_id', timeseries_id) - # datetime needs timezone information - df = df.tz_localize('UTC') - buf = StringIO() - df.to_csv(buf, header=False) - buf.pos = 0 - buf.seek(0) - fake_conn = engine.raw_connection() - fake_cur = fake_conn.cursor() - try: - fake_cur.copy_from(buf, 'data', sep=',', columns=('datetime','value','flags','timeseries_id')) - fake_conn.commit() - message = "Data successfully inserted." - status_code = 200 - except: - e = sys.exc_info()[0] - message = "An error occurred in data insertion: %s" % (e,) + if timeseries: + timeseries_id = timeseries.id + # open SpooledTemporaryFile, skip header (and also try to insert timeseries_id!) + df = pd.read_csv(input_handle.file, comment='#', header=None, sep=';',names=["time","value","flags"],parse_dates=["time"],index_col="time") + # substract timeshift to convert data to UTC + df.index = df.index - timeoffset + # now insert the timeseries_id to the end of the data frame + df.insert(2, 'timeseries_id', timeseries_id) + # datetime needs timezone information + df = df.tz_localize('UTC') + buf = StringIO() + df.to_csv(buf, header=False) + buf.pos = 0 + buf.seek(0) + fake_conn = engine.raw_connection() + fake_cur = fake_conn.cursor() + try: + fake_cur.copy_from(buf, 'data', sep=',', columns=('datetime','value','flags','timeseries_id')) + fake_conn.commit() + message = 'Data successfully inserted.' + status_code = 200 + except: + e = sys.exc_info()[0] + message = 'An error occurred in data insertion: %s' % (e,) + status_code = 400 + else: + message = f'Timeseries not found for station {station_code.strip()}, variable {variable_name}, label {label.strip()}' status_code = 400 return JSONResponse(status_code=status_code, content=message) diff --git a/toardb/data/data.py b/toardb/data/data.py index 55064f6006c1f8120f85eddba5bf7b803235ffd1..c8a76b03b518dbb9cda1a2e80ef8a5048666034b 100644 --- a/toardb/data/data.py +++ b/toardb/data/data.py @@ -60,9 +60,12 @@ async def create_data(file: UploadFile = File(...), db: Session = Depends(get_db # we want to upload a whole file! # response = crud.create_data(db, engine, input_handle=file) - msg = response.body.decode('utf-8') - msg2 = '"An error occurred in data insertion: <class \'psycopg2.errors.UniqueViolation\'>"' - if (msg == msg2): - raise HTTPException(status_code=400, detail="Data for timeseries already registered.") + if response.status_code != 200: + msg = response.body.decode('utf-8') + # try to parse error messages from DBS (to be more understandable) + msg2 = '"An error occurred in data insertion: <class \'psycopg2.errors.UniqueViolation\'>"' + if (msg == msg2): + msg = 'Data for timeseries already registered.' + raise HTTPException(status_code=400, detail=msg) return response diff --git a/toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat b/toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat new file mode 120000 index 0000000000000000000000000000000000000000..f5d7591f322953d39b22f8e7d2a3c262a3d85d5e --- /dev/null +++ b/toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat @@ -0,0 +1 @@ +toluene_SDZ54421_2012_2012_v1-0.dat \ No newline at end of file diff --git a/toardb/data/test_data.py b/toardb/data/test_data.py index ef26e109144f6042d4282a53b0c4fe89171c3347..30915342376035c6f070ef03cd470be53f17f332 100644 --- a/toardb/data/test_data.py +++ b/toardb/data/test_data.py @@ -185,3 +185,12 @@ class TestApps: assert response.status_code == expected_status_code expected_resp = {'detail': 'Data for timeseries already registered.'} assert response.json() == expected_resp + + + def test_insert_missing_id(self, client, db): + response = client.post("/data/", files={"file": open("toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat", "rb")}) + expected_status_code = 400 + assert response.status_code == expected_status_code + print (response.json()) + expected_resp = {'detail': '"Timeseries not found for station SDZ54421, variable o3, label CMA"'} + assert response.json() == expected_resp