#6: new version for transfering stationmeta (saving before crash)

imports_from_TOAR1/get_stations_meta.sh

+#!/bin/bash
+# example script for transfering all UBA stations
+# (country='Germany' and network_name='UBA')
+# numid (now: id) will change --> log mapping: old_id <-> new_id
+#
+# Since UBA stations are the first to be inserted, they are unique.
+# Therefore, no check, if station exists already in database is done.
+# --> next version has to check this
+# --> station_id (and -- if different -- station_local_id)
+#     can also be inserted into codes with no further checks
+#
+# TBD: list of allowed station_countries
+#      (same holds for timezones!)
+# TBD: rename nightlight --> avg_nightlight
+#      (does this also hold for population density?)
+#
+# matching fields:
+# old                                 | new                                  | new table
+# ====================================|======================================|==========================|
+# network_name                        | pointing to programmes               | stationmeta_programmes   |
+# station_id                          | codes                                | stationmeta_core         |
+# station_local_id                    | codes                                | stationmeta_core         |
+# station_type                        | type_of_environment                  | stationmeta_core         |
+# station_type_of_area                | type_of_area                         | stationmeta_core         |
+# station_category                    | additional_metadata                  | stationmeta_core         |
+# station_name                        | name                                 | stationmeta_core         |
+# station_country                     | country                              | stationmeta_core         |
+# station_state                       | state                                | stationmeta_core         |
+# station_lon                         | coordinates                          | stationmeta_core         |
+# station_lat                         | coordinates                          | stationmeta_core         |
+# station_alt                         | coordinates                          | stationmeta_core         |
+# station_reported_alt                | additional_metadata                  | stationmeta_core         |???
+# station_alt_flag                    | additional_metadata                  | stationmeta_core         |???
+# station_google_alt                  | additional_metadata                  | stationmeta_core         |???
+# google_resolution                   | additional_metadata                  | stationmeta_core         |???
+# station_coordinate_status           | coordinate_validation_status         | stationmeta_core         |
+# station_timezone                    | timezone                             | stationmeta_core         |
+# station_landcover_description       | additional_metadata                  | stationmeta_core         |???
+# station_max_population_density_5km  | additional_metadata                  | stationmeta_core         |???
+# station_nightlight_5km              | nightlight_5km_year2013              | stationmeta_global       |
+# station_nightlight_1km              | nightlight_1km_year2013              | stationmeta_global       |
+# station_max_nightlight_25km         | max_nightlight_25km_year2013         | stationmeta_global       |
+# station_climatic_zone               | climatic_zone                        | stationmeta_global       |
+# station_wheat_production            | wheat_production_year2000            | stationmeta_global       |
+# station_rice_production             | rice_production_year2000             | stationmeta_global       |
+# station_nox_emissions               | edgar_htap_v2_nox_emissions_year2010 | stationmeta_global       |
+# station_omi_no2_column              | omi_no2_column_years2011to2015       | stationmeta_global       |
+# station_toar_category               | toar1_category                       | stationmeta_global       |
+# station_htap_region                 | htap_region_tier1                    | stationmeta_global       |
+# station_etopo_alt                   | etopo_alt                            | stationmeta_global       |
+# station_etopo_min_alt_5km           | etopo_min_alt_5km                    | stationmeta_global       |
+# station_etopo_relative_alt          | etopo_relative_alt                   | stationmeta_global       |
+# station_dominant_landcover          | dominant_landcover_year2012          | stationmeta_global       |
+# station_population_density          | population_density_year2010          | stationmeta_global       |
+# station_max_population_density_25km | max_population_density_25km_year2010 | stationmeta_global       |
+# station_comments                    | pointing to annotations              | stationmeta_annotations  |
+#
+# converting old station_coordinates to PostGIS POINT structure
+# converting type_of_environment to controlled vocabulary (ST_vocabulary)
+# converting type_of_area to controlled vocabulary (TA_vocabulary)
+# converting climatic_zone to controlled vocabulary (CZ_vocabulary)
+# converting coordinate_validation_status to controlled vocabulary (CV_vocabulary)
+# converting toar1_category to controlled vocabulary (TC_vocabulary)
+# converting htap_region_tier1 to controlled vocabulary (TR_vocabulary)
+# converting dominant_landcover_year2012 to controlled vocabulary (DL_vocabulary)
+#
+# new (not available in old DB):
+# - coordinate_validation_date (now set to NOW())
+# - station_coordinate_validator_id (now set to 1 (superuser: sschroeder))
+#
+# author: s.schroeder@fz-juelich.de
+# date:   2020-07-05
+
+# just for now: do one station after the other (with manual checks)
+# surface_observations_toar=> select min(numid), max(numid) from stations WHERE network_name='UBA';
+#  min  |  max
+#  ------+-------
+#  3336 | 19316
+
+#psql -h zam10131.zam.kfa-juelich.de -U s.schroeder surface_observations_toar -W -c "\COPY (select numid from stations WHERE network_name='UBA' ORDER BY numid) TO 'UBA_stations.txt'"
+#for STATION in `cat UBA_stations.txt`
+ORDERED_COLUMNS="numid,network_name,station_id,station_local_id,station_type,station_type_of_area,station_category,\
+                 station_name,station_country,station_state,station_lon,station_lat,station_alt,station_timezone,\
+                 station_nightlight_5km,station_climatic_zone,station_wheat_production,station_rice_production,\
+                 station_nox_emissions,station_omi_no2_column,station_toar_category,station_htap_region,station_reported_alt,\
+                 station_alt_flag,station_coordinate_status,station_google_alt,station_etopo_alt,station_etopo_min_alt_5km,\
+                 station_etopo_relative_alt,station_dominant_landcover,station_landcover_description,station_max_nightlight_25km,\
+                 station_max_population_density_25km,station_nightlight_1km,station_population_density,google_resolution,\
+                 station_comments,station_max_population_density_5km"
+for STATION in 3336
+do
+    COUNTER=0
+    while IFS='|' read -a row
+    do
+    if [  $COUNTER -eq 2 ]
+    then
+        NUMID=${row[0]}
+#       echo "NUMID: $NUMID"
+        NETWORK_NAME=${row[1]}
+#       echo "NETWORK_NAME: $NETWORK_NAME"
+        STATION_ID=${row[2]}
+#       echo "STATION_ID: $STATION_ID"
+        STATION_LOCAL_ID=${row[3]}
+#       echo "STATION_LOCAL_ID: $STATION_LOCAL_ID"
+        STATION_TYPE=${row[4]}
+#       echo "STATION_TYPE: $STATION_TYPE"
+        STATION_TYPE_OF_AREA=${row[5]}
+#       echo "STATION_TYPE_OF_AREA: $STATION_TYPE_OF_AREA"
+        STATION_CATEGORY=${row[6]}
+#       echo "STATION_CATEGORY: $STATION_CATEGORY"
+        STATION_NAME=${row[7]}
+#       echo "STATION_NAME: $STATION_NAME"
+        STATION_COUNTRY=${row[8]}
+#       echo "STATION_COUNTRY: $STATION_COUNTRY"
+        STATION_STATE=${row[9]}
+#       echo "STATION_STATE: $STATION_STATE"
+        STATION_LON=${row[10]}
+#       echo "STATION_LON: $STATION_LON"
+        STATION_LAT=${row[11]}
+#       echo "STATION_LAT: $STATION_LAT"
+        STATION_ALT=${row[12]}
+#       echo "STATION_ALT: $STATION_ALT"
+        STATION_TIMEZONE=${row[13]}
+#       echo "STATION_TIMEZONE: $STATION_TIMEZONE"
+        STATION_NIGHTLIGHT_5KM=${row[14]}
+#       echo "STATION_NIGHTLIGHT_5KM: $STATION_NIGHTLIGHT_5KM"
+        STATION_CLIMATIC_ZONE=${row[15]}
+#       echo "STATION_CLIMATIC_ZONE: $STATION_CLIMATIC_ZONE"
+        STATION_WHEAT_PRODUCTION=${row[16]}
+#       echo "STATION_WHEAT_PRODUCTION: $STATION_WHEAT_PRODUCTION"
+        STATION_RICE_PRODUCTION=${row[17]}
+#       echo "STATION_RICE_PRODUCTION: $STATION_RICE_PRODUCTION"
+        STATION_NOX_EMISSIONS=${row[18]}
+#       echo "STATION_NOX_EMISSIONS: $STATION_NOX_EMISSIONS"
+        STATION_OMI_NO2_COLUMN=${row[19]}
+#       echo "STATION_OMI_NO2_COLUMN: $STATION_OMI_NO2_COLUMN"
+        STATION_TOAR_CATEGORY=${row[20]}
+#       echo "STATION_TOAR_CATEGORY: $STATION_TOAR_CATEGORY"
+        STATION_HTAP_REGION=${row[21]}
+#       echo "STATION_HTAP_REGION: $STATION_HTAP_REGION"
+        STATION_REPORTED_ALT=${row[22]}
+#       echo "STATION_REPORTED_ALT: $STATION_REPORTED_ALT"
+        STATION_ALT_FLAG=${row[23]}
+#       echo "STATION_ALT_FLAG: $STATION_ALT_FLAG"
+        STATION_COORDINATE_STATUS=${row[24]}
+#       echo "STATION_COORDINATE_STATUS: $STATION_COORDINATE_STATUS"
+        STATION_GOOGLE_ALT=${row[25]}
+#       echo "STATION_GOOGLE_ALT: $STATION_GOOGLE_ALT"
+        STATION_ETOPO_ALT=${row[26]}
+#       echo "STATION_ETOPO_ALT: $STATION_ETOPO_ALT"
+        STATION_ETOPO_MIN_ALT_5KM=${row[27]}
+#       echo "STATION_ETOPO_MIN_ALT_5KM: $STATION_ETOPO_MIN_ALT_5KM"
+        STATION_ETOPO_RELATIVE_ALT=${row[28]}
+#       echo "STATION_ETOPO_RELATIVE_ALT: $STATION_ETOPO_RELATIVE_ALT"
+        STATION_DOMINANT_LANDCOVER=${row[29]}
+#       echo "STATION_DOMINANT_LANDCOVER: $STATION_DOMINANT_LANDCOVER"
+        STATION_LANDCOVER_DESCRIPTION=${row[30]}
+#       echo "STATION_LANDCOVER_DESCRIPTION: $STATION_LANDCOVER_DESCRIPTION"
+        STATION_MAX_NIGHTLIGHT_25KM=${row[31]}
+#       echo "STATION_MAX_NIGHTLIGHT_25KM: $STATION_MAX_NIGHTLIGHT_25KM"
+        STATION_MAX_POPULATION_DENSITY_25KM=${row[32]}
+#       echo "STATION_MAX_POPULATION_DENSITY_25KM: $STATION_MAX_POPULATION_DENSITY_25KM"
+        STATION_NIGHTLIGHT_1KM=${row[33]}
+#       echo "STATION_NIGHTLIGHT_1KM: $STATION_NIGHTLIGHT_1KM"
+        STATION_POPULATION_DENSITY=${row[34]}
+#       echo "STATION_POPULATION_DENSITY: $STATION_POPULATION_DENSITY"
+        GOOGLE_RESOLUTION=${row[35]}
+#       echo "GOOGLE_RESOLUTION: $GOOGLE_RESOLUTION"
+        STATION_COMMENTS=${row[36]}
+#       echo "STATION_COMMENTS: $STATION_COMMENTS"
+        STATION_MAX_POPULATION_DENSITY_5KM=${row[37]}
+#       echo "STATION_MAX_POPULATION_DENSITY_5KM: $STATION_MAX_POPULATION_DENSITY_5KM"
+    fi
+    let COUNTER=COUNTER+1
+done < <(echo "SELECT ${ORDERED_COLUMNS} FROM stations WHERE numid=${STATION}" | psql -h zam10131.zam.kfa-juelich.de -d surface_observations_toar -U s.schroeder)
+#   res=`psql -h zam10131.zam.kfa-juelich.de -d surface_observations_toar -U s.schroeder -c "SELECT * FROM stations WHERE numid=${STATION}"`
+#   echo $res
+#   MIN_DATETIME=$(echo $res | cut -f2 -d'|' | cut -f4,5 -d' ')
+#   HELPMAX=$(echo $res | cut -f3 -d'|' | cut -f1 -d'(')
+#   MAX_DATETIME=$(echo $HELPMAX)
+#   psql -h zam10131.zam.kfa-juelich.de -d surface_observations_toar -U s.schroeder -c "UPDATE parameter_series SET data_start_date='${MIN_DATETIME}', data_end_date='${MAX_DATETIME}' WHERE id=${SERIES}"
+done
+
+# first test on test database
+#psql -h localhost -U toaradmin toardb -W -c "\COPY stationmeta_core FROM 'stations_China_20200605_import.txt'"
+#psql -h localhost -U toaradmin toardb -W -c "UPDATE stationmeta_core SET country=TRIM(country),codes=TRIM(codes),name=TRIM(name),state=TRIM(state);"
+# operational database
+##psql -h zam10116.zam.kfa-juelich.de -U s.schroeder toardb_v2 -W -c "\COPY stationmeta_core FROM 'stations_China_20200605_import.txt'"
+##psql -h zam10116.zam.kfa-juelich.de -U s.schroeder toardb_v2 -W -c "UPDATE stationmeta_core SET country=TRIM(country),codes=TRIM(codes),name=TRIM(name),state=TRIM(state);"