From 5009a3f248214a651cf144872835e15a47188cb7 Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Thu, 4 Jun 2020 11:51:17 +0200
Subject: [PATCH] #1: added another test (timeseries not existing); tried to
have more user-friendly error messages; still struggling why I get doubled
quotation marks now
---
toardb/data/crud.py | 53 ++++++++++---------
toardb/data/data.py | 11 ++--
.../fixtures/o3_SDZ54421_2012_2012_v1-0.dat | 1 +
toardb/data/test_data.py | 9 ++++
4 files changed, 46 insertions(+), 28 deletions(-)
create mode 120000 toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat
diff --git a/toardb/data/crud.py b/toardb/data/crud.py
index b7a2d68..461cc46 100644
--- a/toardb/data/crud.py
+++ b/toardb/data/crud.py
@@ -54,32 +54,37 @@ def create_data(db: Session, engine: Engine, input_handle: UploadFile = File(...
timeoffset = dt.timedelta(hours=float(line.split(':')[1]))
prev, pos = pos, f.tell()
f.seek(prev)
- stationmeta_core = get_stationmeta_core(db=db,station_code=station_id)
+ station_code = station_id
+ stationmeta_core = get_stationmeta_core(db=db,station_code=station_code)
station_id = stationmeta_core.id
timeseries = get_timeseries_by_unique_constraints(db=db,station_id=station_id,variable_id=variable_id,label=label)
- timeseries_id = timeseries.id
- # open SpooledTemporaryFile, skip header (and also try to insert timeseries_id!)
- df = pd.read_csv(input_handle.file, comment='#', header=None, sep=';',names=["time","value","flags"],parse_dates=["time"],index_col="time")
- # substract timeshift to convert data to UTC
- df.index = df.index - timeoffset
- # now insert the timeseries_id to the end of the data frame
- df.insert(2, 'timeseries_id', timeseries_id)
- # datetime needs timezone information
- df = df.tz_localize('UTC')
- buf = StringIO()
- df.to_csv(buf, header=False)
- buf.pos = 0
- buf.seek(0)
- fake_conn = engine.raw_connection()
- fake_cur = fake_conn.cursor()
- try:
- fake_cur.copy_from(buf, 'data', sep=',', columns=('datetime','value','flags','timeseries_id'))
- fake_conn.commit()
- message = "Data successfully inserted."
- status_code = 200
- except:
- e = sys.exc_info()[0]
- message = "An error occurred in data insertion: %s" % (e,)
+ if timeseries:
+ timeseries_id = timeseries.id
+ # open SpooledTemporaryFile, skip header (and also try to insert timeseries_id!)
+ df = pd.read_csv(input_handle.file, comment='#', header=None, sep=';',names=["time","value","flags"],parse_dates=["time"],index_col="time")
+ # substract timeshift to convert data to UTC
+ df.index = df.index - timeoffset
+ # now insert the timeseries_id to the end of the data frame
+ df.insert(2, 'timeseries_id', timeseries_id)
+ # datetime needs timezone information
+ df = df.tz_localize('UTC')
+ buf = StringIO()
+ df.to_csv(buf, header=False)
+ buf.pos = 0
+ buf.seek(0)
+ fake_conn = engine.raw_connection()
+ fake_cur = fake_conn.cursor()
+ try:
+ fake_cur.copy_from(buf, 'data', sep=',', columns=('datetime','value','flags','timeseries_id'))
+ fake_conn.commit()
+ message = 'Data successfully inserted.'
+ status_code = 200
+ except:
+ e = sys.exc_info()[0]
+ message = 'An error occurred in data insertion: %s' % (e,)
+ status_code = 400
+ else:
+ message = f'Timeseries not found for station {station_code.strip()}, variable {variable_name}, label {label.strip()}'
status_code = 400
return JSONResponse(status_code=status_code, content=message)
diff --git a/toardb/data/data.py b/toardb/data/data.py
index 55064f6..c8a76b0 100644
--- a/toardb/data/data.py
+++ b/toardb/data/data.py
@@ -60,9 +60,12 @@ async def create_data(file: UploadFile = File(...), db: Session = Depends(get_db
# we want to upload a whole file!
#
response = crud.create_data(db, engine, input_handle=file)
- msg = response.body.decode('utf-8')
- msg2 = '"An error occurred in data insertion: <class \'psycopg2.errors.UniqueViolation\'>"'
- if (msg == msg2):
- raise HTTPException(status_code=400, detail="Data for timeseries already registered.")
+ if response.status_code != 200:
+ msg = response.body.decode('utf-8')
+ # try to parse error messages from DBS (to be more understandable)
+ msg2 = '"An error occurred in data insertion: <class \'psycopg2.errors.UniqueViolation\'>"'
+ if (msg == msg2):
+ msg = 'Data for timeseries already registered.'
+ raise HTTPException(status_code=400, detail=msg)
return response
diff --git a/toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat b/toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat
new file mode 120000
index 0000000..f5d7591
--- /dev/null
+++ b/toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat
@@ -0,0 +1 @@
+toluene_SDZ54421_2012_2012_v1-0.dat
\ No newline at end of file
diff --git a/toardb/data/test_data.py b/toardb/data/test_data.py
index ef26e10..3091534 100644
--- a/toardb/data/test_data.py
+++ b/toardb/data/test_data.py
@@ -185,3 +185,12 @@ class TestApps:
assert response.status_code == expected_status_code
expected_resp = {'detail': 'Data for timeseries already registered.'}
assert response.json() == expected_resp
+
+
+ def test_insert_missing_id(self, client, db):
+ response = client.post("/data/", files={"file": open("toardb/data/fixtures/o3_SDZ54421_2012_2012_v1-0.dat", "rb")})
+ expected_status_code = 400
+ assert response.status_code == expected_status_code
+ print (response.json())
+ expected_resp = {'detail': '"Timeseries not found for station SDZ54421, variable o3, label CMA"'}
+ assert response.json() == expected_resp
--
GitLab