diff --git a/tests/fixtures/data/staging_data.json b/tests/fixtures/data/staging_data.json
new file mode 100644
index 0000000000000000000000000000000000000000..bf436b4c9ecc4843b362a154921d8ac80a725e73
--- /dev/null
+++ b/tests/fixtures/data/staging_data.json
@@ -0,0 +1,128 @@
+[
+ {
+ "datetime":"2013-12-16 21:00:00+00",
+ "value":21.581,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-16 22:00:00+00",
+ "value":13.734,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-16 23:00:00+00",
+ "value":13.734,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 00:00:00+00",
+ "value":7.848,
+ "flags":11,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 01:00:00+00",
+ "value":15.696,
+ "flags":11,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 02:00:00+00",
+ "value":11.772,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 03:00:00+00",
+ "value":13.734,
+ "flags":12,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 04:00:00+00",
+ "value":19.62,
+ "flags":11,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 05:00:00+00",
+ "value":15.696,
+ "flags":12,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2013-12-17 06:00:00+00",
+ "value":5.886,
+ "flags":10,
+ "timeseries_id":2,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-16 21:00:00+00",
+ "value":21.581,
+ "flags":10,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-16 22:00:00+00",
+ "value":13.734,
+ "flags":11,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-16 23:00:00+00",
+ "value":13.734,
+ "flags":11,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 00:00:00+00",
+ "value":7.848,
+ "flags":12,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 01:00:00+00",
+ "value":15.696,
+ "flags":12,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 02:00:00+00",
+ "value":11.772,
+ "flags":10,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 03:00:00+00",
+ "value":13.734,
+ "flags":11,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ },
+ {
+ "datetime":"2012-12-17 04:00:00+00",
+ "value":19.62,
+ "flags":10,
+ "timeseries_id":1,
+ "version":"000001.000000.00000000000000"
+ }
+]
diff --git a/tests/fixtures/toardb_pytest.psql b/tests/fixtures/toardb_pytest.psql
index cad82623e3ac3183cf302d385b3dfd7956d1b229..372960d688a8bb1177149bff228bc55cddf13b05 100644
--- a/tests/fixtures/toardb_pytest.psql
+++ b/tests/fixtures/toardb_pytest.psql
@@ -2439,6 +2439,24 @@ CREATE TABLE IF NOT EXISTS public.data_archive (
ALTER TABLE public.data_archive OWNER TO postgres;
+CREATE SCHEMA staging;
+
+--
+-- Name: data; Type: TABLE; Schema: staging; Owner: postgres
+--
+
+CREATE TABLE IF NOT EXISTS staging.data (
+ datetime timestamp with time zone NOT NULL,
+ value double precision NOT NULL,
+ flags integer NOT NULL,
+ timeseries_id integer NOT NULL,
+ version character(28) DEFAULT '000001.000000.00000000000000'::bpchar NOT NULL,
+ CONSTRAINT data_archive_flags_check CHECK ((flags >= 0))
+);
+
+
+ALTER TABLE staging.data OWNER TO postgres;
+
--
-- Name: organisations; Type: TABLE; Schema: public; Owner: postgres
--
diff --git a/tests/test_data.py b/tests/test_data.py
index fe796170d025fccc490e2a4aa4629f188289e080..8686a13bf9731bbf12cd96294d4beb857508a931 100644
--- a/tests/test_data.py
+++ b/tests/test_data.py
@@ -193,6 +193,15 @@ class TestApps:
db.add(new_data)
db.commit()
db.refresh(new_data)
+ infilename = "tests/fixtures/data/staging_data.json"
+ with open(infilename) as f:
+ metajson=json.load(f)
+ for entry in metajson:
+ new_data = Data (**entry)
+ fake_cur.execute(("INSERT INTO staging.data(datetime, value, flags, timeseries_id, version) "
+ f" VALUES('{new_data.datetime}', {new_data.value}, {new_data.flags}, "
+ f"{new_data.timeseries_id}, '{new_data.version}');"))
+ fake_conn.commit()
def test_get_data(self, client, db):
@@ -539,6 +548,114 @@ class TestApps:
{'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'}]}
assert response.json() == expected_response
+
+ def test_get_data_with_staging(self, client, db):
+ with patch('toardb.timeseries.crud.dt.datetime', FixedDatetime):
+ response = client.get("/data/timeseries_with_staging/id/2")
+ expected_status_code = 200
+ assert response.status_code == expected_status_code
+ expected_response = {'metadata': {'id': 2,
+ 'label': 'CMA',
+ 'order': 1,
+ 'sampling_frequency': 'hourly',
+ 'aggregation': 'mean',
+ 'data_start_date': '2003-09-07T15:30:00+00:00',
+ 'data_end_date': '2016-12-31T14:30:00+00:00',
+ 'data_origin': 'instrument',
+ 'data_origin_type': 'measurement',
+ 'provider_version': 'N/A',
+ 'sampling_height': 7.0,
+ 'additional_metadata':
+ {'original_units': {'since_19740101000000': 'nmol/mol'},
+ 'measurement_method': 'uv_abs',
+ 'absorption_cross_section': 0,
+ 'ebas_metadata_19740101000000_29y':
+ {'Submitter': 'Unknown, Lady, lady.unknown@unknown.com, '
+ 'some long division name, SHORT, , '
+ '111 Streetname, , zipcode, Boulder, CO, USA',
+ 'Data level': '2',
+ 'Frameworks': 'GAW-WDCRG NOAA-ESRL',
+ 'Station code': 'XXX',
+ 'Station name': 'Secret'}},
+ 'doi': '',
+ 'coverage': -1.0,
+ 'station': {'id': 3,
+ 'codes': ['China_test8'],
+ 'name': 'Test_China',
+ 'coordinates': {'lat': 36.256, 'lng': 117.106, 'alt': 1534.0},
+ 'coordinate_validation_status': 'not checked',
+ 'country': 'China',
+ 'state': 'Shandong Sheng',
+ 'type': 'unknown',
+ 'type_of_area': 'unknown',
+ 'timezone': 'Asia/Shanghai',
+ 'additional_metadata': {},
+ 'roles': [],
+ 'annotations': [],
+ 'aux_images': [],
+ 'aux_docs': [],
+ 'aux_urls': [],
+ 'globalmeta': None,
+ 'changelog': []},
+ 'variable': {'name': 'o3',
+ 'longname': 'ozone',
+ 'displayname': 'Ozone',
+ 'cf_standardname': 'mole_fraction_of_ozone_in_air',
+ 'units': 'nmol mol-1',
+ 'chemical_formula': 'O3',
+ 'id': 5},
+ 'programme': {'id': 0,
+ 'name': '',
+ 'longname': '',
+ 'homepage': '',
+ 'description': ''},
+ 'roles': [{'id': 1,
+ 'role': 'resource provider',
+ 'status': 'active',
+ 'contact': {'id': 5,
+ 'person': None,
+ 'organisation':
+ {'id': 2,
+ 'name': 'FZJ',
+ 'longname': 'Forschungszentrum Jülich',
+ 'kind': 'research',
+ 'city': 'Jülich',
+ 'postcode': '52425',
+ 'street_address': 'Wilhelm-Johnen-Straße',
+ 'country': 'Germany',
+ 'homepage': 'https://www.fz-juelich.de',
+ 'contact_url': 'mailto:toar-data@fz-juelich.de'}}}],
+ 'changelog': None,
+ 'citation': 'Forschungszentrum Jülich: time series of o3 at '
+ 'Test_China, accessed from the TOAR database on '
+ '2023-07-28 12:00:00',
+ 'attribution': 'Test-Attributions to be announced',
+ 'license': 'This data is published under a Creative Commons '
+ 'Attribution 4.0 International (CC BY 4.0). '
+ 'https://creativecommons.org/licenses/by/4.0/'
+ },
+ 'data': [{'datetime': '2012-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'OK validated modified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'OK validated modified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'OK validated QC passed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2012-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'OK validated verified', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-16T21:00:00+00:00', 'value': 21.581, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-16T22:00:00+00:00', 'value': 13.734, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-16T23:00:00+00:00', 'value': 13.734, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T00:00:00+00:00', 'value': 7.848, 'flags': 'questionable validated unconfirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T01:00:00+00:00', 'value': 15.696, 'flags': 'questionable validated unconfirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T02:00:00+00:00', 'value': 11.772, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T03:00:00+00:00', 'value': 13.734, 'flags': 'questionable validated flagged', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T04:00:00+00:00', 'value': 19.62, 'flags': 'questionable validated unconfirmed', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T05:00:00+00:00', 'value': 15.696, 'flags': 'questionable validated flagged', 'timeseries_id': 2, 'version': '1.0'},
+ {'datetime': '2013-12-17T06:00:00+00:00', 'value': 5.886, 'flags': 'questionable validated confirmed', 'timeseries_id': 2, 'version': '1.0'}]
+ }
+ assert response.json() == expected_response
+
+
def test_create_data_record(self, client, db):
response = client.post("/data/timeseries/record/?series_id=2&datetime=2021-08-23%2015:00:00&value=67.3&flag=OK&version=000001.000001.00000000000000")
expected_status_code = 200
diff --git a/toardb/test_base.py b/toardb/test_base.py
index 0dd44ebc068ac4e9ee05cb8e8be65c7cb8757af8..ad331ce9e768e6c25acfc9440024e6637cde8375 100644
--- a/toardb/test_base.py
+++ b/toardb/test_base.py
@@ -69,6 +69,7 @@ def test_db_session():
# otherwiese all tables from "toar_controlled_vocabulary" will get lost!
if not tbl.name.endswith("_vocabulary"):
_db_conn.execute(tbl.delete())
+ _db_conn.execute("DELETE FROM staging.data;")
fake_conn = _db_conn.raw_connection()
fake_cur = fake_conn.cursor()
fake_cur.execute("ALTER TABLE timeseries_changelog ALTER COLUMN datetime SET DEFAULT now();")