From 8bd6331fbe6d2352d3e4438dc83f9d891d4027ac Mon Sep 17 00:00:00 2001
From: schroeder5 <s.schroeder@fz-juelich.de>
Date: Tue, 7 Jul 2020 11:01:48 +0200
Subject: [PATCH] #6: transfer script for data (timeseriesID changed from old
DB to new DB)
---
imports_from_TOAR1/get_data.sh | 70 +++++++++++++++++++++-------------
1 file changed, 44 insertions(+), 26 deletions(-)
diff --git a/imports_from_TOAR1/get_data.sh b/imports_from_TOAR1/get_data.sh
index f008424..9b80de3 100755
--- a/imports_from_TOAR1/get_data.sh
+++ b/imports_from_TOAR1/get_data.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# script for transfering data (old DB: spec_samplingInterval) to table data(new DB)
+# script for transfering UBA ozone data (old DB: o3_hourly) to table data(new DB)
#
# new design of data:
# Table "public.data"
@@ -15,32 +15,50 @@
# - datetime -- with time zone included ==> This means copy within UTC!
#
# author: s.schroeder@fz-juelich.de
-# date: 2020-06-05
+# date: 2020-07-05
-HOURLY_SPEC="co no pm1 o3 no2 so2 ox aswdir pm10 rn ch4 wdir pm2p5 nox temp wspeed press cloudcover pblheight relhum totprecip u v albedo aswdifu humidity irradiance"
-EVENT_SPEC="benzene toluene ethane propane mpxylene oxylene"
-MONTHLY_SPEC="ch4 co"
-
-for sampling in hourly event monthly
+while read -a line
do
- if [ "$sampling" = "hourly" ]
- then
- speclist=$HOURLY_SPEC
- elif [ "$sampling" = "event" ]
- then
- speclist=$EVENT_SPEC
- else
- speclist=$MONTHLY_SPEC
- fi
- for SPEC in $speclist
- do
- for ID in ... # here id-list (SELECT all!)
- echo "extracting data for timeseries ${ID}"
- psql -h zam10131.zam.kfa-juelich.de -U s.schroeder surface_observations_toar -W -c "\COPY (select id, CONCAT(datetime, '+00'), value, flag from ${SPEC}_${sampling} WHERE id=${ID}) TO 'data_export.txt'"
+ OLD_TIMESERIESID=${line[0]}
+ NEW_TIMESERIESID=${line[1]}
+ VARNAME=${line[2]}
+ SAMPLING=${line[3]}
+ echo "extracting data for timeseries ${OLD_TIMESERIESID} --> ${NEW_TIMESERIESID}"
+ psql -h zam10131.zam.kfa-juelich.de -U s.schroeder surface_observations_toar -c "\COPY (select '$NEW_TIMESERIESID', CONCAT(datetime, '+00'), value, CONCAT(flag,'+',preliminary) from ${VARNAME}_${SAMPLING} WHERE id=${OLD_TIMESERIESID}) TO 'data_export.txt'"
+
+ # convert old flags to new system of controlled vocabulary
+ # --------------------------------------------------------
+ # old flags: Definition of the data quality flag values according to WMO code table 033 020.
+ # See WMO manual on codes
+ # (http://www.wmo.int/pages/prog/www/WMOCodes/WMO306_vI2/VolumeI.2.html).
+ # FLAG[0] = "OK"
+ # FLAG[1] = "inconsistent"
+ # FLAG[2] = "doubtful"
+ # FLAG[3] = "wrong"
+ # FLAG[4] = "not_checked"
+ # FLAG[5] = "changed" # not quite clear what this means wrt quality...
+ # FLAG[6] = "estimated"
+ # FLAG[7] = "missing_value" # we do not store missing values in DB
+ #
+ # ==> (in combination with preliminary flag):
+ # 0+f --> OK (0)
+ # 0+t --> OKPreliminary (1)
+ # --> OKModified (2)
+ # --> OKPreliminaryModified (3)
+ # 1+t --> Inconsistent (4)
+ # 1+f --> InconsistentPreliminary (5)
+ # 2+t --> Doubtful (6)
+ # 2+f --> DoubtfulPreliminary (7)
+ # --> DoubtfulModified (8)
+ # --> DoubtfulPreliminaryModified (9)
+ # 3+t --> Wrong (10)
+ # 3+f --> WrongPreliminary (11)
+ # 4+f --> NotCheckedPreliminary (12)
+ # 5+t --> Changed (13)
+ # 6+t --> Estimated (14)
+ # 7+ --> MissingValue (15) --> will never happen
- #do some adaptions to the data (especially flags=flag+prelimary+... (see above))
- #...
- psql -h localhost -U django toar2 -W -c "\COPY data (timeseries_id, date_time, value, flags) FROM 'data_export.txt'"
- done
-done
+ sed -i 's/0+f/0/; s/0+t/1/; s/1+t/4/; s/1+f/5/; s/2+t/6/; s/2+f/7/; s/3+t/10/; s/3+f/11/; s/4+f/12/; s/5+t/13/; s/6+t/14/' data_export.txt
+ psql -h zam10116.zam.kfa-juelich.de -U s.schroeder -d toardb_v2 -c "\COPY data (timeseries_id, datetime, value, flags) FROM 'data_export.txt'"
+done < timeseries_oldID_newID_var_samp.txt
--
GitLab