Merge branch 'lukas_issue004_feat_new-app-stable-night-lights' into 'develop'

Lukas issue004 feat new app stable night lights

See merge request toar/toar-location-services!4

.gitignore

@@ -42,3 +42,7 @@ Thumbs.db
 # data folder #
 ###############
 /data/
+
+# check plot folder #
+#####################
+/plots/
\ No newline at end of file

population-density/population_file_extraction.py

@@ -8,8 +8,8 @@ Author: Martin Schultz, FZ Juelich (04 May 2016)
 import numpy as np
 import os
 from toar_location_services.settings import DATA_DIR, DEBUG, USE_DUMMY_POPULATION_DATA
+import matplotlib.pyplot as plt
 
-# import matplotlib.pyplot as plt
 
 def read_proxydata(filename, dummy=DEBUG and USE_DUMMY_POPULATION_DATA):
     """Read the ascii file and return the data array together with
@@ -52,11 +52,16 @@ def read_proxydata(filename, dummy=DEBUG and USE_DUMMY_POPULATION_DATA):
             row = np.array([float(x) for x in line.split()], dtype='f4')
             data[i, :] = row
 
+        # correct missing values
+        data[data == float(missval)] = np.nan
+
     logdata = data.copy()
     logdata[logdata <= 1.e-4] = 1.e-4
-    # plt.contourf(lonvec, latvec, np.log10(logdata))
-    # plt.savefig('global_population_density.png')
-    # plt.close()
+
+    if DEBUG:
+        plt.contourf(lonvec, latvec, np.log10(logdata))
+        plt.savefig('../plots/global_population_density.png')
+        plt.close()
 
     # set metadata
     boundingbox = [lon0, lat0, lonvec.max(), latvec.max()]

population-density/tests.py

 from django.test import TestCase
 
 # Create your tests here.
+
+from population_file_extraction import read_proxydata
+
+
+FILENAME = "population.txt"
+
+read_proxydata(FILENAME, dummy=False)

requirements.txt

@@ -11,4 +11,4 @@ requests>=2.19.1
 pip>=10.0.1
 setuptools>=39.1.0
 wheel>=0.32.2
-
+matplotlib>=3.1.0

stable_night_lights/admin.py

+from django.contrib import admin
+
+# Register your models here.

stable_night_lights/apps.py

+from django.apps import AppConfig
+
+
+class StableNightLightsConfig(AppConfig):
+    name = 'stable_night_lights'

stable_night_lights/models.py

+from django.db import models
+
+# Create your models here.

stable_night_lights/nightlights_file_extraction.py

+#!/usr/bin/python
+"""
+Load stable night lights data from file or use dummy data.
+
+..todo: implementation of RASDAMAN access
+
+Author: Lukas Leufen, FZ Juelich (21th May 2019)
+"""
+import numpy as np
+import os
+from toar_location_services.settings import DATA_DIR, DEBUG, USE_DUMMY_STABLE_NIGHT_LIGHTS_DATA
+from django.contrib.gis.gdal import GDALRaster
+import matplotlib.pyplot as plt
+# import gdal
+# from gdalconst import GA_ReadOnly
+
+
+def read_proxydata(filename, dummy=DEBUG and USE_DUMMY_STABLE_NIGHT_LIGHTS_DATA):
+    """
+    PLACEHOLDER
+    expand this routine, if you want to use real and not dummy data
+    """
+    if dummy:
+        return create_dummy_data()
+
+    NIGHTLIGHTS_FILE = os.path.join(DATA_DIR, filename)
+
+    if DEBUG:
+        print("DATA_DIR =  ", DATA_DIR, "...")
+        print("Opening ", NIGHTLIGHTS_FILE, "...")
+    rst = GDALRaster(NIGHTLIGHTS_FILE, write=False)
+    lon0, lat0 = rst.origin  # top left corner
+    xmin, ymin, xmax, ymax = rst.extent  # xmin=lon0, ymin, xmax, ymax=lat0
+    cols = rst.width
+    rows = rst.height
+    dlon, dlat = rst.scale  # xscale=dlon, yscale=dlat
+
+    # construct data array and lonvec, latvec
+    data = rst.bands[0].data().astype(np.float)
+    nodata_value = rst.bands[0].nodata_value
+    data[data == nodata_value] = np.nan
+    lonvec = np.linspace(xmin, xmax, cols)
+    latvec = np.linspace(ymin, ymax, rows)
+
+    # check if data is flipped
+    lat1 = lat0 + dlat * rows
+    if lat1 < lat0:
+        lat0, lat1 = lat1, lat0
+        latvec = np.flipud(latvec)
+
+    # set metadata
+    boundingbox = [lon0, latvec.min(), lonvec.max(), lat0]
+
+    # plot data
+    if DEBUG:
+        plt.contourf(lonvec, latvec, data)
+        plt.savefig('../plots/global_nighttime_lights.png')
+        plt.close()
+
+    #
+    datainfo = {'size': (rows, cols), 'resolution': (np.abs(dlon), np.abs(dlat)),
+                'boundingbox': boundingbox}
+    return lonvec, latvec, data, datainfo
+
+
+def create_dummy_data():
+    """generate some small dummy data set for testing of other services
+    This avoids loading of a large file
+    Eventually this should become obsolete when we have rasdaman running..."""
+    lonvec = np.array([4., 5., 6.])
+    latvec = np.array([52., 53., 54.])
+    data = np.array([[100., 100., 105.], [200., 250., 300.], [600., 700., 800.]])
+    datainfo = {'size': (3, 3), 'resolution': (1., 1.),
+                'boundingbox': [4., 52., 6., 54.]}
+    msg = "#DEBUG: Using dummy data for stable night lights!"
+    if DEBUG:
+        print(msg)
+    else:
+        raise UserWarning(msg)
+    return lonvec, latvec, data, datainfo

stable_night_lights/serializers.py

+"""serializer for stable night lights data
+"""
+
+from collections import OrderedDict
+import datetime as dt
+from rest_framework.serializers import BaseSerializer
+
+
+# helper functions
+
+def get_provenance(obj):
+    """construct provenance information on stable night lights dataset"""
+    prov = OrderedDict([
+        ('dataset_name', 'night time lights'),
+        ('dataset_description', 'Year 2013 Nighttime lights brightness values from NOAA DMSP. Resolution? Morde '
+                                'Details?'),
+        ('data_source', """Year 2013 Nighttime lights brightness values from NOAA DMSP"""),
+        ('datacenter_url', '?'),
+        ('download_date', '?'),
+        ('timestamp', dt.datetime.now().isoformat())
+    ])
+    return prov
+
+# serializer classes
+
+class AggSerializer(BaseSerializer):
+    """ see http://www.django-rest-framework.org/api-guide/serializers/#baseserializer """
+
+    def to_representation(self, obj):
+        """takes dictionary-like obj and returns geojson compliant structure"""
+        agg_function = obj['agg_function']
+        val = obj[agg_function]
+
+        # build GeoJSON response with 'provenance' extension
+        # ToDo (probably in views): change content-type to application/vnd.geo+json
+        # ToDo: enable support for different output formats
+        # format properties depending on 'by_direction' (vector or not)
+        try:
+            vlength = len(val)
+        except TypeError:
+            vlength = 1
+        if vlength > 1:
+            properties = OrderedDict([
+                ('agg_function', agg_function),
+                ('many', True),
+                (agg_function, OrderedDict([
+                    (d, v) for d, v in zip(obj['direction'], val)
+                ])),
+                ('units', 'm'),
+                ('radius', obj['radius']),
+            ])
+            if obj['direction'] is None:
+                properties.pop('direction')
+        else:
+            properties = OrderedDict([
+                ('agg_function', agg_function),
+                ('many', False),
+                (agg_function, val),
+                ('units', 'm'),
+                ('radius', obj['radius']),
+                ('direction', obj['direction']),
+            ])
+            if obj['direction'] is None:
+                properties.pop('direction')
+
+        response = OrderedDict([
+            ('type', 'Feature'),
+            ('geometry', OrderedDict([
+                ('type', 'Point'),
+                ('coordinates', [obj['lon'], obj['lat']]),
+                ])),
+            ('properties', properties),
+            ('provenance', get_provenance(obj)),
+        ])
+
+        return response

stable_night_lights/tests.py

+from django.test import TestCase
+
+# Create your tests here.
+
+from nightlights_file_extraction import read_proxydata
+
+
+FILENAME = "stable_lights_europe.tif"
+
+
+read_proxydata(FILENAME, dummy=False)

stable_night_lights/urls.py

+from django.conf.urls import url
+from .views import NightTimeLightsView
+
+urlpatterns = [
+    url(r'^$', NightTimeLightsView.as_view()),
+    ]

stable_night_lights/views.py

+import datetime as dt
+import numpy as np
+from collections import OrderedDict
+from rest_framework.views import APIView
+from rest_framework.response import Response
+
+from toar_location_services.settings import DEBUG
+from .nightlights_file_extraction import read_proxydata
+from .serializers import AggSerializer
+from utils.views_commons import get_query_params, get_agg_function
+from utils.geoutils import Directions
+from utils.extraction_tools import extract_value, extract_value_stats
+
+FILENAME = "stable_lights_europe.tif"
+
+lonvec, latvec, data, datainfo = read_proxydata(FILENAME)
+
+if DEBUG:
+    print("File %s successfully loaded" % FILENAME, datainfo)
+
+
+class NightTimeLightsView(APIView):
+
+    def _extract(self, lat, lon, radius, agg, direction, by_direction):
+        """perform actual extraction of desired quantity"""
+        print('**by_direction:', by_direction, '** radius, agg = ', radius, agg)
+        if agg is not None and radius is not None and radius > 0.:
+            agg_function = get_agg_function(agg)
+            min_angle = None
+            max_angle = None
+            if by_direction:
+                result = np.zeros((16,))
+                direction = Directions.LABELS
+                for i, d in enumerate(Directions.LABELS):
+                    min_angle, max_angle = Directions.edges(d)
+                    # ToDo: once we serve the data via rasdaman the calls with directions should use
+                    # a polygon query for efficiency reasons.
+                    result[i] = extract_value_stats(lonvec, latvec, data, lon, lat, default_value=-999.,
+                                                 out_of_bounds_value=0., min_valid=0., max_valid=2.e6,
+                                                 radius=radius, min_angle=min_angle, max_angle=max_angle,
+                                                 agg=agg_function)
+            else:
+                if direction is not None:
+                    min_angle, max_angle = Directions.edges(direction)
+                    # ToDo: once we serve the data via rasdaman the calls with directions should use
+                    # a polygon query for efficiency reasons.
+                result = extract_value_stats(lonvec, latvec, data, lon, lat, default_value=-999.,
+                                             out_of_bounds_value=0., min_valid=0., max_valid=2.e6,
+                                             radius=radius, min_angle=min_angle, max_angle=max_angle,
+                                             agg=agg_function)
+        else:
+            agg = 'value'
+            result = extract_value(lonvec, latvec, data, lon, lat, default_value=-999.,
+                                   out_of_bounds_value=0., min_valid=0., max_valid=2.e6)
+        # return data, also return agg and direction as they may have been overwritten
+        return result, agg, direction
+
+    def get(self, request, format=None):
+        """process GET requests for topography-tandem-X app
+
+        returns a Geo-JSON response with information about the topography at or
+        around a point location.
+
+        required arguments:
+        lat: lattiude in degrees_north
+        lng: longitude in degrees_east (can be either -180 to 180 or 0 to 360)
+
+        optional arguments:
+        radius: search radius in m. See settings.py for default and max allowed values.
+            Without 'agg', the radius defaults to None and the population density at the
+            point location is returned.
+        agg: method of aggregation for data around point location. See settings.py for
+            default method. Only evaluated if radius > 0. Allowed methods are mean,
+            min, max, median, and NN-percentile (see views_commons.py)
+        direction: return data aggregation in one direction (wind sector) only.
+            Direction must be given as wind sector (e.g. 'N', 'NNE', 'NE', etc.).
+        by_direction: if True, data are returned as vector with one value aggregated
+            over each of 16 wind directions.
+        """
+        lat, lon, radius, agg, direction, by_direction = get_query_params(request.query_params,
+                                                 ['lat', 'lon', 'radius', 'agg', 'direction', 'by_direction'])
+        result, agg, direction = self._extract(lat, lon, radius, agg, direction, by_direction)
+
+        rawdata = OrderedDict([
+            ("lat", lat),
+            ("lon", lon),
+            ("radius", radius),
+            ("direction", direction),
+            ("agg_function", agg),
+            (agg, result),
+        ])
+        response = AggSerializer(rawdata).data
+        return Response(response)
+

toar_location_services/settings.py

@@ -42,7 +42,8 @@ INSTALLED_APPS = [
     'rest_framework',
     'major-roads',
     'population-density',
-    'topography-tandem-x'
+    'topography-tandem-x',
+    'stable_night_lights'
 ]
 
 MIDDLEWARE = [
@@ -143,4 +144,5 @@ DEFAULT_LON = 4.9003
 USE_LOCAL_OVERPASS_DATA = False
 USE_DUMMY_POPULATION_DATA = True
 USE_DUMMY_TOPOGRAPHY_TANDEM_DATA = False
+USE_DUMMY_STABLE_NIGHT_LIGHTS_DATA = False
 

toar_location_services/urls.py

@@ -7,5 +7,6 @@ urlpatterns = [
     url('admin/', admin.site.urls),
     url(r'major-roads/', include('major-roads.urls'), name='major-roads'),
     url(r'population-density/', include('population-density.urls'), name='population-density-density'),
-    url(r'topography-tandem-x/', include('topography-tandem-x.urls'), name='topography-tandem-x')
+    url(r'topography-tandem-x/', include('topography-tandem-x.urls'), name='topography-tandem-x'),
+    url(r'stable_night_lights/', include('stable_night_lights.urls'), name='stable_night_lights')
 ]

toar_location_services/views.py

@@ -15,6 +15,7 @@ class LocationServicesRootView(APIView):
                 ('major-roads', request.build_absolute_uri()+'major-roads/'),
                 ('population-density', request.build_absolute_uri()+'population-density/'),
                 ('topography-tandem-x', request.build_absolute_uri()+'topography-tandem-x/'),
+                ('stable_night_lights', request.build_absolute_uri()+'stable_night_lights/'),
             ]),
         ]
         # add admin view if staff user