diff --git a/imports_from_TOAR1/get_timeseries_meta.sh b/imports_from_TOAR1/get_timeseries_meta.sh
index 5a13b70b74fbe294cc3017f23a8592572aef6ecd..7ee6726981ca210842cc7ba820c26cb41917ccd2 100755
--- a/imports_from_TOAR1/get_timeseries_meta.sh
+++ b/imports_from_TOAR1/get_timeseries_meta.sh
@@ -50,13 +50,22 @@
# - sampling height (now set to 2 (m))
# - programme_id (now set to 0 (None-programme))
# - source (UBA: 0 (Model), if label == REA, 1 (Measurement) else -- MM_vocabulary)
+# It was double-checked that all data with parameter_contributor_shortname='MIUB' have label='REA'!
# - kind (UBA: 1 (Government) -- OK_vocabulary)
# - role (UBA: 1 (PrincipalInvestigator) and 3 (Contributor) -- RC_vocabulary)
# - status (UBA: 2 (unknown for PI) and 0 (active for Contributor) -- RS_vocabulary)
#
+# for UBA: use parameter_attribute for LABEL information
+# --> measurements: LABEL=''
+# --> models: LABEL='REA'
+# ==> they have to be distinguishable (AND unique constraint on (station_id,variable_id,label)
+#
# author: s.schroeder@fz-juelich.de
# date: 2020-07-05
+PARAMETER_NAME=temp
+SAMPLING=hourly
+
ORDERED_COLUMNS="id,station_numid,parameter_label,parameter_name,parameter_attribute,parameter_sampling_type,\
parameter_measurement_method,parameter_original_units,parameter_calibration,parameter_contributor_shortname,\
parameter_contributor,parameter_contributor_country,parameter_dataset_type,parameter_status,comments,\
@@ -73,10 +82,11 @@ PI_ROLE=1
PI_STATUS=2
CONTRIBUTOR_ROLE=3
CONTRIBUTOR_STATUS=0
-LABEL=''
-VARIABLE_ID=5
AGGREGATION=1
-SOURCE=1
+
+res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "SELECT id FROM variables\
+ WHERE name='$PARAMETER_NAME';"`
+VARIABLE_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
while read -a line
do
@@ -89,54 +99,57 @@ do
if [ $COUNTER -eq 2 ]
then
ID=${row[0]}
-# echo "ID: $ID"
+ echo "ID: $ID"
STATION_NUMID=${row[1]}
-# echo "STATION_NUMID: $STATION_NUMID"
+ echo "STATION_NUMID: $STATION_NUMID"
PARAMETER_LABEL=${row[2]}
-# echo "PARAMETER_LABEL: $PARAMETER_LABEL"
- PARAMETER_NAME=${row[3]}
+ echo "PARAMETER_LABEL: $PARAMETER_LABEL"
+# We already set PARAMETER_NAME manually!
+# The following two lines can be used when debugging...
+# (and we do not want to overwrite PARAMETER_NAME with untrimmed value)
+# PARAMETER_NAME=${row[3]}
# echo "PARAMETER_NAME: $PARAMETER_NAME"
PARAMETER_ATTRIBUTE=${row[4]}
-# echo "PARAMETER_ATTRIBUTE: $PARAMETER_ATTRIBUTE"
+ echo "PARAMETER_ATTRIBUTE: $PARAMETER_ATTRIBUTE"
PARAMETER_SAMPLING_TYPE=${row[5]}
-# echo "PARAMETER_SAMPLING_TYPE: $PARAMETER_SAMPLING_TYPE"
+ echo "PARAMETER_SAMPLING_TYPE: $PARAMETER_SAMPLING_TYPE"
PARAMETER_MEASUREMENT_METHOD=${row[6]}
-# echo "PARAMETER_MEASUREMENT_METHOD: $PARAMETER_MEASUREMENT_METHOD"
+ echo "PARAMETER_MEASUREMENT_METHOD: $PARAMETER_MEASUREMENT_METHOD"
PARAMETER_ORIGINAL_UNITS=${row[7]}
-# echo "PARAMETER_ORIGINAL_UNITS: $PARAMETER_ORIGINAL_UNITS"
+ echo "PARAMETER_ORIGINAL_UNITS: $PARAMETER_ORIGINAL_UNITS"
PARAMETER_CALIBRATION=${row[8]}
-# echo "PARAMETER_CALIBRATION: $PARAMETER_CALIBRATION"
+ echo "PARAMETER_CALIBRATION: $PARAMETER_CALIBRATION"
PARAMETER_CONTRIBUTOR_SHORTNAME=${row[9]}
-# echo "PARAMETER_CONTRIBUTOR_SHORTNAME: $PARAMETER_CONTRIBUTOR_SHORTNAME"
+ echo "PARAMETER_CONTRIBUTOR_SHORTNAME: $PARAMETER_CONTRIBUTOR_SHORTNAME"
PARAMETER_CONTRIBUTOR=${row[10]}
-# echo "PARAMETER_CONTRIBUTOR: $PARAMETER_CONTRIBUTOR"
+ echo "PARAMETER_CONTRIBUTOR: $PARAMETER_CONTRIBUTOR"
PARAMETER_CONTRIBUTOR_COUNTRY=${row[11]}
-# echo "PARAMETER_CONTRIBUTOR_COUNTRY: $PARAMETER_CONTRIBUTOR_COUNTRY"
+ echo "PARAMETER_CONTRIBUTOR_COUNTRY: $PARAMETER_CONTRIBUTOR_COUNTRY"
PARAMETER_DATASET_TYPE=${row[12]}
-# echo "PARAMETER_DATASET_TYPE: $PARAMETER_DATASET_TYPE"
+ echo "PARAMETER_DATASET_TYPE: $PARAMETER_DATASET_TYPE"
PARAMETER_STATUS=${row[13]}
-# echo "PARAMETER_STATUS: $PARAMETER_STATUS"
+ echo "PARAMETER_STATUS: $PARAMETER_STATUS"
COMMENTS=${row[14]}
-# echo "COMMENTS: $COMMENTS"
+ echo "COMMENTS: $COMMENTS"
CREATION_DATE=${row[15]}
-# echo "CREATION_DATE: $CREATION_DATE"
+ echo "CREATION_DATE: $CREATION_DATE"
MODIFICATION_DATE=${row[16]}
-# echo "MODIFICATION_DATE: $MODIFICATION_DATE"
+ echo "MODIFICATION_DATE: $MODIFICATION_DATE"
DATA_START_DATE=${row[17]}
-# echo "DATA_START_DATE: $DATA_START_DATE"
+ echo "DATA_START_DATE: $DATA_START_DATE"
DATA_END_DATE=${row[18]}
-# echo "DATA_END_DATE: $DATA_END_DATE"
+ echo "DATA_END_DATE: $DATA_END_DATE"
PARAMETER_PI=${row[19]}
-# echo "PARAMETER_PI: $PARAMETER_PI"
+ echo "PARAMETER_PI: $PARAMETER_PI"
PARAMETER_PI_EMAIL=${row[20]}
-# echo "PARAMETER_PI_EMAIL: $PARAMETER_PI_EMAIL"
+ echo "PARAMETER_PI_EMAIL: $PARAMETER_PI_EMAIL"
PARAMETER_INSTRUMENT_MANUFACTURER=${row[21]}
-# echo "PARAMETER_INSTRUMENT_MANUFACTURER: $PARAMETER_INSTRUMENT_MANUFACTURER"
+ echo "PARAMETER_INSTRUMENT_MANUFACTURER: $PARAMETER_INSTRUMENT_MANUFACTURER"
PARAMETER_INSTRUMENT_MODEL=${row[22]}
-# echo "PARAMETER_INSTRUMENT_MODEL: $PARAMETER_INSTRUMENT_MODEL"
+ echo "PARAMETER_INSTRUMENT_MODEL: $PARAMETER_INSTRUMENT_MODEL"
fi
let COUNTER=COUNTER+1
- done < <(echo "SELECT ${ORDERED_COLUMNS} FROM parameter_series WHERE station_numid=${OLD_STATIONID} AND parameter_name='o3'" | psql -h zam10131.zam.kfa-juelich.de -d surface_observations_toar -U s.schroeder)
+ done < <(echo "SELECT ${ORDERED_COLUMNS} FROM parameter_series WHERE station_numid=${OLD_STATIONID} AND parameter_name='${PARAMETER_NAME}' AND parameter_dataset_type='${SAMPLING}'" | psql -h zam10131.zam.kfa-juelich.de -d surface_observations_toar -U s.schroeder)
if [ "$ID" != "(0 rows)" ]
then
@@ -147,21 +160,33 @@ do
# 1. a) persons
# =============
- # no personal contacts given for UBA data!
- # only model data from REA give personal contacts!
- # ozone data is only measured from UBA!
-# EMAIL=`echo -n "${PARAMETER_PI_EMAIL//[[:space:]]/}"`
-# NAME=`echo -n "${PARAMETER_PI//[[:space:]]/}"`
-# PHONE=''
-# ISPRIVATE=True
-
-# res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO persons\
-# (name,email,phone,isprivate) VALUES
-# ('$NAME','$EMAIL','$PHONE',$ISPRIVATE)\
-# RETURNING id;"`
-# PERSON_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
- PERSON_ID=-1
-
+ EMAIL=`echo -n "${PARAMETER_PI_EMAIL//[[:space:]]/}"`
+ NAME=`echo -e $PARAMETER_PI | awk '{$1=$1};1'`
+ PHONE=''
+ ISPRIVATE=True
+
+ # at least for UBA data: email='unknown', name='unknown' given for undefined
+ # ==> to be checked for other providers!!!
+ PERSON_ID=-1
+ if [ "$EMAIL" != "unknown" ] || [ "$NAME" != "unknown" ]
+ then
+ # check, whether person exists already in database
+# res=`psql -h localhost -d toardb -U toaradmin -c "SELECT id FROM persons\
+ res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "SELECT id FROM persons\
+ WHERE name='$NAME' AND EMAIL='$EMAIL';"`
+ PERS_EXISTED=1
+ PERSON_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
+ if [ "$PERSON_ID" == "(0" ]
+ then
+ PERS_EXISTED=0
+ res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO persons\
+ (name,email,phone,isprivate) VALUES
+ ('$NAME','$EMAIL','$PHONE',$ISPRIVATE)\
+ RETURNING id;"`
+ PERSON_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
+ fi
+ fi
+
# 1. b) organisations
# ===================
@@ -173,21 +198,25 @@ do
STREET_ADDRESS=''
HOMEPAGE=''
- # check, whether organisation exists already in database
-# res=`psql -h localhost -d toardb -U toaradmin -c "SELECT id FROM organisations\
- res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "SELECT id FROM organisations\
- WHERE name='$NAME' AND longname='$LONGNAME';"`
- ORG_EXISTED=1
- ORGANISATION_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
- if [ "$ORGANISATION_ID" == "(0" ]
- then
- ORG_EXISTED=0
-# res=`psql -h localhost -d toardb -U toaradmin -c "INSERT INTO organisations\
- res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO organisations\
- (name,longname,kind,city,postcode,street_address,country,homepage) VALUES \
- ('$NAME','$LONGNAME',$KIND,'$CITY','$POSTCODE','$STREET_ADDRESS','$COUNTRY','$HOMEPAGE') \
- RETURNING id;"`
+ ORGANISATION_ID=-1
+ if [ "$NAME" != "unknown" ] || [ "$LONGNAME" != "unknown" ]
+ then
+ # check, whether organisation exists already in database
+# res=`psql -h localhost -d toardb -U toaradmin -c "SELECT id FROM organisations\
+ res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "SELECT id FROM organisations\
+ WHERE name='$NAME' AND longname='$LONGNAME';"`
+ ORG_EXISTED=1
ORGANISATION_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
+ if [ "$ORGANISATION_ID" == "(0" ]
+ then
+ ORG_EXISTED=0
+# res=`psql -h localhost -d toardb -U toaradmin -c "INSERT INTO organisations\
+ res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO organisations\
+ (name,longname,kind,city,postcode,street_address,country,homepage) VALUES \
+ ('$NAME','$LONGNAME',$KIND,'$CITY','$POSTCODE','$STREET_ADDRESS','$COUNTRY','$HOMEPAGE') \
+ RETURNING id;"`
+ ORGANISATION_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
+ fi
fi
# 1. c) contacts
@@ -195,11 +224,18 @@ do
if [ $PERSON_ID -ne -1 ]
then
-# res=`psql -h localhost -d toardb -U toaradmin -c "INSERT INTO contacts\
- res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO contacts\
- (person_id,organisation_id) VALUES \
- ($PERSON_ID,0) \
- RETURNING id;"`
+ if [ $PERS_EXISTED -eq 1 ]
+ then
+# res=`psql -h localhost -d toardb -U toaradmin -c "SELECT id FROM contacts\
+ res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "SELECT id FROM contacts\
+ WHERE person_id=$PERSON_ID;"`
+ else
+# res=`psql -h localhost -d toardb -U toaradmin -c "INSERT INTO contacts\
+ res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO contacts\
+ (person_id,organisation_id) VALUES \
+ ($PERSON_ID,0) \
+ RETURNING id;"`
+ fi
PI_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
fi
@@ -286,12 +322,21 @@ do
\"parameter_instrument_model\":\"$TRIMMED_PARAMETER_INSTRUMENT_MODEL\"}"
DATE_ADDED=$CREATION_DATE
DATE_MODIFIED=$MODIFICATION_DATE
+
+ # what about lower_case?????
+
if [[ $PARAMETER_MEASUREMENT_METHOD == *"UV"* ]]
then
MEASUREMENT_METHOD=0
else
MEASUREMENT_METHOD=1
fi
+ SOURCE=1
+ if [ "$TRIMMED_PARAMETER_ATTRIBUTE" == "REA" ]
+ then
+ SOURCE=0
+ fi
+ LABEL=$TRIMMED_PARAMETER_ATTRIBUTE
# res=`psql -h localhost -d toardb -U toaradmin -c "INSERT INTO timeseries \
res=`psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO timeseries\
(label,\"order\",access_rights,sampling_frequency,aggregation,data_start_date,data_end_date,sampling_height,\
@@ -301,7 +346,7 @@ do
RETURNING id;"`
# log new id
NEW_ID=$(echo $res | cut -f2 | cut -f3 -d' ')
- echo "$ID $NEW_ID" >>timeseries_oldID_newID.txt
+ echo "$ID $NEW_ID $PARAMETER_NAME $SAMPLING" >>"timeseries_oldID_newID_${PARAMETER_NAME}_${SAMPLING}.txt"
# 4. many-to-many relations
# =========================
@@ -309,11 +354,20 @@ do
# 4. a) timeseries_timeseries_roles
# =================================
- # for UBA: only Contributor (see above)
-# psql -h localhost -d toardb -U toaradmin -c "INSERT INTO timeseries_timeseries_roles\
- psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO timeseries_timeseries_roles\
- (timeseries_id,role_id) VALUES \
- ($NEW_ID,$ROLE2);"
+ if [ $PERSON_ID -ne -1 ]
+ then
+# psql -h localhost -d toardb -U toaradmin -c "INSERT INTO timeseries_timeseries_roles\
+ psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO timeseries_timeseries_roles\
+ (timeseries_id,role_id) VALUES \
+ ($NEW_ID,$ROLE1);"
+ fi
+ if [ $ORGANISATION_ID -ne -1 ]
+ then
+# psql -h localhost -d toardb -U toaradmin -c "INSERT INTO timeseries_timeseries_roles\
+ psql -h zam10116.zam.kfa-juelich.de -d toardb_v2 -U s.schroeder -c "INSERT INTO timeseries_timeseries_roles\
+ (timeseries_id,role_id) VALUES \
+ ($NEW_ID,$ROLE2);"
+ fi
# 4. b) timeseries_timeseries_annotations