Commit 8ed692ad authored by Niklas Selke's avatar Niklas Selke
Browse files

fixed the population number and population density services

parent 62d5d99b
Pipeline #94018 passed with stages
in 50 seconds
......@@ -54,7 +54,7 @@ INSTALLED_APPS = [
# 'services.omi_no2',
# 'services.htap_regions_tier1',
# 'services.htap_regions_tier2',
'services.human_settlement',
'services.population_number',
# 'services.built_up_area',
# 'services.population_density_ghs',
]
......@@ -185,7 +185,7 @@ USE_DUMMY_CLIMATIC_ZONES_DATA = False
USE_DUMMY_NOX_EMISSION_DATA = False
USE_DUMMY_OMI_NO2_DATA = False
USE_DUMMY_HTAP_REGIONS_DATA = False
USE_DUMMY_HUMAN_SETTLEMENT_DATA = False
USE_DUMMY_POPULATION_NUMBER_DATA = False
USE_DUMMY_BUILT_UP_AREA_DATA = False
USE_SMALL_DATA = True
PLOT_DATA = False
......
......@@ -18,7 +18,7 @@ urlpatterns = [
# re_path(r'omi_no2/', include('services.omi_no2.urls'), name='omi_no2'),
# re_path(r'htap_regions_tier1/', include('services.htap_regions_tier1.urls'), name='htap_regions_tier1'),
# re_path(r'htap_regions_tier2/', include('services.htap_regions_tier2.urls'), name='htap_regions_tier2'),
re_path(r'human_settlement/', include('services.human_settlement.urls'), name='human_settlement'),
re_path(r'population_number/', include('services.population_number.urls'), name='population_number'),
# re_path(r'built_up_area/', include('services.built_up_area.urls'), name='built_up_area'),
# re_path(r'population_density_ghs/', include('services.population_density_ghs.urls'), name='population_density_ghs')
]
......
......@@ -17,8 +17,9 @@ class LocationServicesRootView(APIView):
- for topography_srtm:
relative= [True|False]
- for human_settlement, stable_nightlights:
year= (for human_settlement: at the moment you must set the year to 1975!)
- for population_number, stable_nightlights:
year= (for population_number: 1975, 1990, 2000, 2015
for stable_nightlights: 1992-2013)
- ecoregions:
description=[True|False]
......@@ -44,7 +45,7 @@ class LocationServicesRootView(APIView):
# ('omi_no2', request.build_absolute_uri()+'omi_no2/'),
# ('htap_regions_tier1', request.build_absolute_uri()+'htap_regions_tier1/'),
# ('htap_regions_tier2', request.build_absolute_uri()+'htap_regions_tier2/'),
('human_settlement', request.build_absolute_uri()+'human_settlement/'),
('population_number', request.build_absolute_uri()+'population_number/'),
# ('built_up_area', request.build_absolute_uri()+'built_up_area/'),
# ('population_density_ghs', request.build_absolute_uri()+'population_density_ghs/')
]),
......
#!/usr/bin/python
"""
Load human settlement data from file or use dummy data.
Load population number data from file or use dummy data.
Author: Lukas Leufen, FZ Juelich (15th October 2019)
"""
import numpy as np
import os
from geo_location_services.settings import DATA_DIR, DEBUG, USE_DUMMY_HUMAN_SETTLEMENT_DATA, PLOT_DATA, TEST
from geo_location_services.settings import DATA_DIR, DEBUG, USE_DUMMY_POPULATION_NUMBER_DATA, PLOT_DATA, TEST
from django.contrib.gis.gdal import GDALRaster
import matplotlib.pyplot as plt
def read_proxydata(filename, dummy=DEBUG and USE_DUMMY_HUMAN_SETTLEMENT_DATA): # pragma: no cover
def read_proxydata(filename, dummy=DEBUG and USE_DUMMY_POPULATION_NUMBER_DATA): # pragma: no cover
"""
PLACEHOLDER
expand this routine, if you want to use real and not dummy data
......@@ -59,7 +59,7 @@ def read_proxydata(filename, dummy=DEBUG and USE_DUMMY_HUMAN_SETTLEMENT_DATA):
cmap = plt.cm.get_cmap('gist_ncar')
cax = ax.contourf(lonvec, latvec, data, levels=[9, 10, 11, 12, 13, 21, 22, 23, 30, 31], cmap=cmap)
plt.colorbar(cax)
plt.savefig('../plots/human_settlement.png')
plt.savefig('../plots/population_number.png')
plt.close()
#
......
from django.apps import AppConfig
class PopulationConfig(AppConfig): # pragma: no cover
class PopulationDensityConfig(AppConfig): # pragma: no cover
name = 'services.population_density'
......@@ -6,39 +6,9 @@ import datetime as dt
from utils.agg_serializer import GeneralAggSerializer
# helper functions
def get_provenance(obj):
"""construct provenance information on population density dataset"""
prov = OrderedDict([
('dataset_name', 'glp10ag.asc'),
('dataset_description', """Gridded Population of the World, Version 3 (GPWv3): Future Estimates, for the year
2010 Data produced by Center for International Earth Science Information Network (CIESIN), Columbia University.
This data contains the UN-adjusted population counts for the year 2010. The raster data are at 2.5 arc-minutes
resolution.
"""),
('data_source', """Center for International Earth Science Information Network (CIESIN), Columbia University;
United Nations Food and Agriculture Programme (FAO); and Centro Internacional de Agricultura Tropical (CIAT).
2005. Gridded Population of the World: Future Estimates (GPWFE). Palisades, NY: Socioeconomic Data and
Applications Center (SEDAC), Columbia University. Available at http://sedac.ciesin.columbia.edu/gpw.
(date of download) """),
('datacenter_url', 'http://sedac.ciesin.columbia.edu/gpw'),
('download_date', '2011-10-19'),
('timestamp', dt.datetime.now().isoformat()),
('Terms of use', """The Trustees of Columbia University in the City of New York and the Centro Internacional de
Agricultura Tropical (CIAT) hold the copyright of this data set. Users are prohibited from any commercial,
non-free resale, or redistribution without explicit written permission from CIESIN and CIAT. Users should
acknowledge CIESIN and CIAT as the source used in the creation of any reports, publications, new data sets,
derived products, or services resulting from the use of this data set. CIESIN and CIAT also request reprints of
any publications and notification of any redistributing efforts.""")
])
return prov
# serializer classes
class AggSerializer(GeneralAggSerializer):
def __init__(self, instance):
instance['unit'] = ''
instance['get_provenance'] = get_provenance
instance['unit'] = '1/km^2'
super(AggSerializer, self).__init__(instance)
import geo_location_services.settings as settings
from django.test import TestCase
from services.population_density.serializers import get_provenance, AggSerializer
from services.population_density.views import PopulationView
from services.population_number.serializers import AggSerializer
from services.population_density.views import PopulationDensityView
from collections import OrderedDict
class TestSerializers(TestCase):
def test_get_provenance(self):
required_elements = ['dataset_name', 'dataset_description', 'data_source', 'datacenter_url', 'download_date',
'timestamp', 'Terms of use']
self.assertListEqual(required_elements, list(get_provenance(None).keys()))
def test_AggSerializer(self):
rawdata = OrderedDict([
("lat", settings.DEFAULT_LAT),
......@@ -25,14 +20,13 @@ class TestSerializers(TestCase):
serializer = AggSerializer(rawdata)
self.assertEqual(serializer.instance["unit"], '')
self.assertEqual(serializer.instance["radius"], 5000)
self.assertTrue(serializer.instance["get_provenance"] is get_provenance)
class TestViews(TestCase):
def test_PopulationView(self):
view = PopulationView()
def test_PopulationDensityView(self):
view = PopulationDensityView()
self.assertEqual(view.min_valid, 0)
self.assertEqual(view.max_valid, 2.e6)
self.assertEqual(view.max_valid, 2.e9)
self.assertTrue(view.AggSerializer is AggSerializer)
self.assertListEqual(view.AGG_ALLOWED, ['mean', 'min', 'max', 'median', 'percentile', 'sum', 'stddev'])
from django.urls import re_path
from .views import PopulationView
from .views import PopulationDensityView
urlpatterns = [
re_path(r'^$', PopulationView.as_view()),
re_path(r'^$', PopulationDensityView.as_view()),
]
......@@ -2,7 +2,7 @@ import math
# from geo_location_services.settings import DEBUG, USE_LOCAL_DATA, USE_SMALL_DATA
# from services.file_extractions.population_density_file_extraction import read_proxydata
from services.human_settlement.views import HumanSettlementView
from services.population_number.views import PopulationNumberView
# if USE_LOCAL_DATA: # pragma: no cover
......@@ -15,11 +15,7 @@ from services.human_settlement.views import HumanSettlementView
# print("File %s successfully loaded" % FILENAME, datainfo)
class PopulationView(HumanSettlementView):
def __init__(self):
HumanSettlementView.__init__(self)
class PopulationDensityView(PopulationNumberView):
def get(self, request, format=None):
"""process GET requests
......@@ -41,7 +37,7 @@ class PopulationView(HumanSettlementView):
direction: return data aggregation in one direction (wind sector) only.
Direction must be given as wind sector (e.g. 'N', 'NNE', 'NE', etc.).
"""
result = HumanSettlementView.get(self, request, format=format)
result = PopulationNumberView.get(self, request, format=format)
properties = result.data["properties"]
radius = properties["radius"]
arcsec = 30 if radius is not None and radius > 10000 else 9
......
from django.apps import AppConfig
class HumanSettlementConfig(AppConfig): # pragma: no cover
name = 'services.human_settlement'
class PopulationNumberConfig(AppConfig): # pragma: no cover
name = 'services.population_number'
"""serializer for human settlement data
"""serializer for population number data
"""
from collections import OrderedDict
......
from django.test import TestCase
from services.human_settlement.serializers import AggSerializer
from services.human_settlement.views import HumanSettlementView
from services.population_number.serializers import AggSerializer
from services.population_number.views import PopulationNumberView
import geo_location_services.settings as settings
from collections import OrderedDict
......@@ -24,11 +24,11 @@ class TestSerializers(TestCase):
class TestViews(TestCase):
def test_HumanSettlementView(self):
view = HumanSettlementView()
def test_PopulationNumberView(self):
view = PopulationNumberView()
self.assertEqual(view.min_valid, 0)
self.assertEqual(view.max_valid, 2.e9)
self.assertTrue(view.AggSerializer is AggSerializer)
self.assertListEqual(view.AGG_ALLOWED, ['mean', 'min', 'max', 'median', 'percentile', 'sum', 'stddev'])
self.assertEqual(view.service_type, "services.human_settlement")
self.assertEqual(view.service_type, "services.population_number")
from django.urls import re_path
from .views import HumanSettlementView
from .views import PopulationNumberView
urlpatterns = [
re_path(r'^$', HumanSettlementView.as_view()),
re_path(r'^$', PopulationNumberView.as_view()),
]
from rest_framework.views import APIView
# from geo_location_services.settings import DEBUG, USE_LOCAL_DATA
# from services.file_extractions.human_settlement_file_extraction import read_proxydata
# from services.file_extractions.population_number_file_extraction import read_proxydata
from .serializers import AggSerializer
from utils.views_commons import CommonViewsWithYear
from .apps import HumanSettlementConfig as Config
from .apps import PopulationNumberConfig as Config
# if USE_LOCAL_DATA: # pragma: no cover
......@@ -13,7 +13,7 @@ from .apps import HumanSettlementConfig as Config
# print("File %s successfully loaded" % FILENAME, datainfo)
class HumanSettlementView(APIView, CommonViewsWithYear):
class PopulationNumberView(APIView, CommonViewsWithYear):
def __init__(self):
......
......@@ -13,8 +13,8 @@ class GeoCube:
'services.topography_srtm': 'topography__srtm',
'services.ecoregions': 'ecoregions_2017__resolve',
'services.climatic_zones': 'climatic_zone__ipcc',
'services.human_settlement': 'population_density__ghsl_09ss',
'services.human_settlement_coarse': 'population_density__ghsl_30ss',
'services.population_number': 'population_number__ghsl_09ss',
'services.population_number_coarse': 'population_number__ghsl_30ss',
# 'rice_production': 'rice',
# 'wheat_production': 'wheat'
}
......@@ -148,7 +148,7 @@ class GeoCube:
self.coverage = self.wcs_alt.getCoverage(**self.param)
def set_query(self, year, lat, lon, radius):
if self.service == 'services.human_settlement' and radius is not None and radius > 10000:
if self.service == 'services.population_number' and radius is not None and radius > 10000:
self.__init__(service=self.service+"_coarse")
res = self.get_resolution(abs_val=True)
if radius is None:
......@@ -220,7 +220,7 @@ class GeoCube:
return lonvec, latvec, data, datainfo
def get_provenance(self):
if self.service in ['services.human_settlement', 'services.human_settlement_coarse']:
if self.service in ['services.population_number', 'services.population_number_coarse']:
message = ("For radius <= 10km a grid with 9 arcsec resolution is used, "
"for radius > 10km a grid with 30 arcsec resolution is used")
self.metadata['provenance'] = {**{'important': message}, **self.metadata['provenance']}
......
......@@ -1019,7 +1019,7 @@ class TestGeoCubeConnection(TestCase):
def test_cube(self):
self.assertEqual(self.cube.service, "services.stable_nightlights")
self.assertEqual(self.cube.identifier, "nighttime_lights__DMSP_OLS")
self.assertEqual(self.cube.identifier, "nighttime_lights__dmsp_ols")
self.assertEqual(self.cube.url, "http://zam10182.zam.kfa-juelich.de/rasdaman/ows")
self.assertEqual(self.cube.wcs_version, "2.0.1")
self.assertDictEqual(self.cube.param, {})
......@@ -1089,12 +1089,12 @@ class TestGeoCubeConnection(TestCase):
def test_set_param(self):
self.assertDictEqual(self.cube.param, {})
self.cube.set_param()
self.assertDictEqual(self.cube.param, {"identifier": ["nighttime_lights__DMSP_OLS"],
self.assertDictEqual(self.cube.param, {"identifier": ["nighttime_lights__dmsp_ols"],
"subsets": [],
"format": "application/netcdf"})
self.cube.set_subset(('Lon', 0, 80))
self.cube.set_param()
self.assertDictEqual(self.cube.param, {"identifier": ["nighttime_lights__DMSP_OLS"],
self.assertDictEqual(self.cube.param, {"identifier": ["nighttime_lights__dmsp_ols"],
"subsets": [('Lon', 0, 80)],
"format": "application/netcdf"})
self.cube.identifier, self.cube.format, self.cube.subsets, self.cube.param = None, None, None, {}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment