diff --git a/imports_from_TOAR1/get_stations_core_meta.sh b/imports_from_TOAR1/get_stations_core_meta_v0.1.sh similarity index 100% rename from imports_from_TOAR1/get_stations_core_meta.sh rename to imports_from_TOAR1/get_stations_core_meta_v0.1.sh diff --git a/imports_from_TOAR1/get_stations_meta.sh b/imports_from_TOAR1/get_stations_meta.sh new file mode 100755 index 0000000000000000000000000000000000000000..44d8326d774116e135cebddeaf865118e9c64b76 --- /dev/null +++ b/imports_from_TOAR1/get_stations_meta.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# example script for transfering all UBA stations +# (country='Germany' and network_name='UBA') +# numid (now: id) will change --> log mapping: old_id <-> new_id +# +# Since UBA stations are the first to be inserted, they are unique. +# Therefore, no check, if station exists already in database is done. +# --> next version has to check this +# --> station_id (and -- if different -- station_local_id) +# can also be inserted into codes with no further checks +# +# TBD: list of allowed station_countries +# (same holds for timezones!) +# TBD: rename nightlight --> avg_nightlight +# (does this also hold for population density?) +# +# matching fields: +# old | new | new table +# ====================================|======================================|==========================| +# network_name | pointing to programmes | stationmeta_programmes | +# station_id | codes | stationmeta_core | +# station_local_id | codes | stationmeta_core | +# station_type | type_of_environment | stationmeta_core | +# station_type_of_area | type_of_area | stationmeta_core | +# station_category | additional_metadata | stationmeta_core | +# station_name | name | stationmeta_core | +# station_country | country | stationmeta_core | +# station_state | state | stationmeta_core | +# station_lon | coordinates | stationmeta_core | +# station_lat | coordinates | stationmeta_core | +# station_alt | coordinates | stationmeta_core | +# station_reported_alt | additional_metadata | stationmeta_core |??? +# station_alt_flag | additional_metadata | stationmeta_core |??? +# station_google_alt | additional_metadata | stationmeta_core |??? +# google_resolution | additional_metadata | stationmeta_core |??? +# station_coordinate_status | coordinate_validation_status | stationmeta_core | +# station_timezone | timezone | stationmeta_core | +# station_landcover_description | additional_metadata | stationmeta_core |??? +# station_max_population_density_5km | additional_metadata | stationmeta_core |??? +# station_nightlight_5km | nightlight_5km_year2013 | stationmeta_global | +# station_nightlight_1km | nightlight_1km_year2013 | stationmeta_global | +# station_max_nightlight_25km | max_nightlight_25km_year2013 | stationmeta_global | +# station_climatic_zone | climatic_zone | stationmeta_global | +# station_wheat_production | wheat_production_year2000 | stationmeta_global | +# station_rice_production | rice_production_year2000 | stationmeta_global | +# station_nox_emissions | edgar_htap_v2_nox_emissions_year2010 | stationmeta_global | +# station_omi_no2_column | omi_no2_column_years2011to2015 | stationmeta_global | +# station_toar_category | toar1_category | stationmeta_global | +# station_htap_region | htap_region_tier1 | stationmeta_global | +# station_etopo_alt | etopo_alt | stationmeta_global | +# station_etopo_min_alt_5km | etopo_min_alt_5km | stationmeta_global | +# station_etopo_relative_alt | etopo_relative_alt | stationmeta_global | +# station_dominant_landcover | dominant_landcover_year2012 | stationmeta_global | +# station_population_density | population_density_year2010 | stationmeta_global | +# station_max_population_density_25km | max_population_density_25km_year2010 | stationmeta_global | +# station_comments | pointing to annotations | stationmeta_annotations | +# +# converting old station_coordinates to PostGIS POINT structure +# converting type_of_environment to controlled vocabulary (ST_vocabulary) +# converting type_of_area to controlled vocabulary (TA_vocabulary) +# converting climatic_zone to controlled vocabulary (CZ_vocabulary) +# converting coordinate_validation_status to controlled vocabulary (CV_vocabulary) +# converting toar1_category to controlled vocabulary (TC_vocabulary) +# converting htap_region_tier1 to controlled vocabulary (TR_vocabulary) +# converting dominant_landcover_year2012 to controlled vocabulary (DL_vocabulary) +# +# new (not available in old DB): +# - coordinate_validation_date (now set to NOW()) +# - station_coordinate_validator_id (now set to 1 (superuser: sschroeder)) +# +# author: s.schroeder@fz-juelich.de +# date: 2020-07-05 + +# just for now: do one station after the other (with manual checks) +# surface_observations_toar=> select min(numid), max(numid) from stations WHERE network_name='UBA'; +# min | max +# ------+------- +# 3336 | 19316 + +#psql -h zam10131.zam.kfa-juelich.de -U s.schroeder surface_observations_toar -W -c "\COPY (select numid from stations WHERE network_name='UBA' ORDER BY numid) TO 'UBA_stations.txt'" +#for STATION in `cat UBA_stations.txt` +ORDERED_COLUMNS="numid,network_name,station_id,station_local_id,station_type,station_type_of_area,station_category,\ + station_name,station_country,station_state,station_lon,station_lat,station_alt,station_timezone,\ + station_nightlight_5km,station_climatic_zone,station_wheat_production,station_rice_production,\ + station_nox_emissions,station_omi_no2_column,station_toar_category,station_htap_region,station_reported_alt,\ + station_alt_flag,station_coordinate_status,station_google_alt,station_etopo_alt,station_etopo_min_alt_5km,\ + station_etopo_relative_alt,station_dominant_landcover,station_landcover_description,station_max_nightlight_25km,\ + station_max_population_density_25km,station_nightlight_1km,station_population_density,google_resolution,\ + station_comments,station_max_population_density_5km" +for STATION in 3336 +do + COUNTER=0 + while IFS='|' read -a row + do + if [ $COUNTER -eq 2 ] + then + NUMID=${row[0]} +# echo "NUMID: $NUMID" + NETWORK_NAME=${row[1]} +# echo "NETWORK_NAME: $NETWORK_NAME" + STATION_ID=${row[2]} +# echo "STATION_ID: $STATION_ID" + STATION_LOCAL_ID=${row[3]} +# echo "STATION_LOCAL_ID: $STATION_LOCAL_ID" + STATION_TYPE=${row[4]} +# echo "STATION_TYPE: $STATION_TYPE" + STATION_TYPE_OF_AREA=${row[5]} +# echo "STATION_TYPE_OF_AREA: $STATION_TYPE_OF_AREA" + STATION_CATEGORY=${row[6]} +# echo "STATION_CATEGORY: $STATION_CATEGORY" + STATION_NAME=${row[7]} +# echo "STATION_NAME: $STATION_NAME" + STATION_COUNTRY=${row[8]} +# echo "STATION_COUNTRY: $STATION_COUNTRY" + STATION_STATE=${row[9]} +# echo "STATION_STATE: $STATION_STATE" + STATION_LON=${row[10]} +# echo "STATION_LON: $STATION_LON" + STATION_LAT=${row[11]} +# echo "STATION_LAT: $STATION_LAT" + STATION_ALT=${row[12]} +# echo "STATION_ALT: $STATION_ALT" + STATION_TIMEZONE=${row[13]} +# echo "STATION_TIMEZONE: $STATION_TIMEZONE" + STATION_NIGHTLIGHT_5KM=${row[14]} +# echo "STATION_NIGHTLIGHT_5KM: $STATION_NIGHTLIGHT_5KM" + STATION_CLIMATIC_ZONE=${row[15]} +# echo "STATION_CLIMATIC_ZONE: $STATION_CLIMATIC_ZONE" + STATION_WHEAT_PRODUCTION=${row[16]} +# echo "STATION_WHEAT_PRODUCTION: $STATION_WHEAT_PRODUCTION" + STATION_RICE_PRODUCTION=${row[17]} +# echo "STATION_RICE_PRODUCTION: $STATION_RICE_PRODUCTION" + STATION_NOX_EMISSIONS=${row[18]} +# echo "STATION_NOX_EMISSIONS: $STATION_NOX_EMISSIONS" + STATION_OMI_NO2_COLUMN=${row[19]} +# echo "STATION_OMI_NO2_COLUMN: $STATION_OMI_NO2_COLUMN" + STATION_TOAR_CATEGORY=${row[20]} +# echo "STATION_TOAR_CATEGORY: $STATION_TOAR_CATEGORY" + STATION_HTAP_REGION=${row[21]} +# echo "STATION_HTAP_REGION: $STATION_HTAP_REGION" + STATION_REPORTED_ALT=${row[22]} +# echo "STATION_REPORTED_ALT: $STATION_REPORTED_ALT" + STATION_ALT_FLAG=${row[23]} +# echo "STATION_ALT_FLAG: $STATION_ALT_FLAG" + STATION_COORDINATE_STATUS=${row[24]} +# echo "STATION_COORDINATE_STATUS: $STATION_COORDINATE_STATUS" + STATION_GOOGLE_ALT=${row[25]} +# echo "STATION_GOOGLE_ALT: $STATION_GOOGLE_ALT" + STATION_ETOPO_ALT=${row[26]} +# echo "STATION_ETOPO_ALT: $STATION_ETOPO_ALT" + STATION_ETOPO_MIN_ALT_5KM=${row[27]} +# echo "STATION_ETOPO_MIN_ALT_5KM: $STATION_ETOPO_MIN_ALT_5KM" + STATION_ETOPO_RELATIVE_ALT=${row[28]} +# echo "STATION_ETOPO_RELATIVE_ALT: $STATION_ETOPO_RELATIVE_ALT" + STATION_DOMINANT_LANDCOVER=${row[29]} +# echo "STATION_DOMINANT_LANDCOVER: $STATION_DOMINANT_LANDCOVER" + STATION_LANDCOVER_DESCRIPTION=${row[30]} +# echo "STATION_LANDCOVER_DESCRIPTION: $STATION_LANDCOVER_DESCRIPTION" + STATION_MAX_NIGHTLIGHT_25KM=${row[31]} +# echo "STATION_MAX_NIGHTLIGHT_25KM: $STATION_MAX_NIGHTLIGHT_25KM" + STATION_MAX_POPULATION_DENSITY_25KM=${row[32]} +# echo "STATION_MAX_POPULATION_DENSITY_25KM: $STATION_MAX_POPULATION_DENSITY_25KM" + STATION_NIGHTLIGHT_1KM=${row[33]} +# echo "STATION_NIGHTLIGHT_1KM: $STATION_NIGHTLIGHT_1KM" + STATION_POPULATION_DENSITY=${row[34]} +# echo "STATION_POPULATION_DENSITY: $STATION_POPULATION_DENSITY" + GOOGLE_RESOLUTION=${row[35]} +# echo "GOOGLE_RESOLUTION: $GOOGLE_RESOLUTION" + STATION_COMMENTS=${row[36]} +# echo "STATION_COMMENTS: $STATION_COMMENTS" + STATION_MAX_POPULATION_DENSITY_5KM=${row[37]} +# echo "STATION_MAX_POPULATION_DENSITY_5KM: $STATION_MAX_POPULATION_DENSITY_5KM" + fi + let COUNTER=COUNTER+1 +done < <(echo "SELECT ${ORDERED_COLUMNS} FROM stations WHERE numid=${STATION}" | psql -h zam10131.zam.kfa-juelich.de -d surface_observations_toar -U s.schroeder) +# res=`psql -h zam10131.zam.kfa-juelich.de -d surface_observations_toar -U s.schroeder -c "SELECT * FROM stations WHERE numid=${STATION}"` +# echo $res +# MIN_DATETIME=$(echo $res | cut -f2 -d'|' | cut -f4,5 -d' ') +# HELPMAX=$(echo $res | cut -f3 -d'|' | cut -f1 -d'(') +# MAX_DATETIME=$(echo $HELPMAX) +# psql -h zam10131.zam.kfa-juelich.de -d surface_observations_toar -U s.schroeder -c "UPDATE parameter_series SET data_start_date='${MIN_DATETIME}', data_end_date='${MAX_DATETIME}' WHERE id=${SERIES}" +done + +# first test on test database +#psql -h localhost -U toaradmin toardb -W -c "\COPY stationmeta_core FROM 'stations_China_20200605_import.txt'" +#psql -h localhost -U toaradmin toardb -W -c "UPDATE stationmeta_core SET country=TRIM(country),codes=TRIM(codes),name=TRIM(name),state=TRIM(state);" +# operational database +##psql -h zam10116.zam.kfa-juelich.de -U s.schroeder toardb_v2 -W -c "\COPY stationmeta_core FROM 'stations_China_20200605_import.txt'" +##psql -h zam10116.zam.kfa-juelich.de -U s.schroeder toardb_v2 -W -c "UPDATE stationmeta_core SET country=TRIM(country),codes=TRIM(codes),name=TRIM(name),state=TRIM(state);"