Skip to content
Snippets Groups Projects
Commit d7699878 authored by Carsten Hinz's avatar Carsten Hinz
Browse files

adopted script to test example class.

parent 29e04e74
No related branches found
No related tags found
3 merge requests!11Creation of first beta release version,!10change in metadata of notebook,!9change in metadata of notebook
Hide whitespace changes
Inline Side-by-side
examples/quality_controll.ipynb
+ 3
− 3
View file @ d7699878
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### Get Dataset from request ### Get Dataset from request
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from datetime import datetime as dt from datetime import datetime as dt
from pathlib import Path from pathlib import Path
import pandas as pd import pandas as pd
import numpy as np import numpy as np
from toargridding.grids import RegularGrid from toargridding.grids import RegularGrid
from toargridding.toar_rest_client import ( from toargridding.toar_rest_client import (
AnalysisServiceDownload, AnalysisServiceDownload,
STATION_LAT, STATION_LAT,
STATION_LON, STATION_LON,
) )
from toargridding.metadata import Metadata, TimeSample, AnalysisRequestResult, Coordinates from toargridding.metadata import Metadata, TimeSample, AnalysisRequestResult, Coordinates
from toargridding.variables import Coordinate from toargridding.variables import Coordinate
from toargridding.contributors import contributionsManager from toargridding.contributors import contributionsManager, contributions_manager_by_name
import logging import logging
from toargridding.defaultLogging import toargridding_defaultLogging from toargridding.defaultLogging import toargridding_defaultLogging
#setup of logging #setup of logging
logger = toargridding_defaultLogging() logger = toargridding_defaultLogging()
logger.addShellLogger(logging.DEBUG) logger.addShellLogger(logging.DEBUG)
logger.logExceptions() logger.logExceptions()
endpoint = "https://toar-data.fz-juelich.de/api/v2/analysis/statistics/" endpoint = "https://toar-data.fz-juelich.de/api/v2/analysis/statistics/"
#starts in directory [path/to/toargridding]/tests #starts in directory [path/to/toargridding]/tests
#maybe adopt the toargridding_base_path for your machine. #maybe adopt the toargridding_base_path for your machine.
toargridding_base_path = Path(".") toargridding_base_path = Path(".")
cache_dir = toargridding_base_path / "cache" cache_dir = toargridding_base_path / "cache"
data_download_dir = toargridding_base_path / "results" data_download_dir = toargridding_base_path / "results"
analysis_service = AnalysisServiceDownload(endpoint, cache_dir, data_download_dir, use_downloaded=True) analysis_service = AnalysisServiceDownload(endpoint, cache_dir, data_download_dir, use_downloaded=True)
my_grid = RegularGrid(1.9, 2.5) my_grid = RegularGrid(1.9, 2.5)
time = TimeSample(dt(2016,1,1), dt(2016,2,28), "daily") time = TimeSample(dt(2016,1,1), dt(2016,2,28), "daily")
metadata = Metadata.construct("mole_fraction_of_ozone_in_air", time, "mean") metadata = Metadata.construct("mole_fraction_of_ozone_in_air", time, "mean")
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# this cell can runs longer than 30minutes # this cell can runs longer than 30minutes
data = analysis_service.get_data(metadata) data = analysis_service.get_data(metadata)
# create contributors endpoint and write result to metadata # create contributors endpoint and write result to metadata
contrib = contributionsManager(metadata.get_id(), data_download_dir) contrib = contributions_manager_by_name(metadata.get_id(), data_download_dir)
contrib.extract_contributors_from_data_frame(data.stations_data) contrib.extract_contributors_from_data_frame(data.stations_data)
metadata.contributors_metadata_field = contrib.setup_contributors_endpoint_for_metadata() metadata.contributors_metadata_field = contrib.setup_contributors_endpoint_for_metadata()
ds = my_grid.as_xarray(data) ds = my_grid.as_xarray(data)
#store dataset #store dataset
ds.to_netcdf(data_download_dir / f"{metadata.get_id()}_{my_grid.get_id()}.nc") ds.to_netcdf(data_download_dir / f"{metadata.get_id()}_by_names_inline_{my_grid.get_id()}.nc")
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### Visual inspection ### Visual inspection
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#calculation of coordinates for plotting #calculation of coordinates for plotting
#especially separation of coordinates with results and without results. #especially separation of coordinates with results and without results.
import cartopy.crs as ccrs import cartopy.crs as ccrs
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import matplotlib.ticker as mticker import matplotlib.ticker as mticker
mean_data = ds["mean"] mean_data = ds["mean"]
clean_coords = data.stations_coords clean_coords = data.stations_coords
all_na = data.stations_data.isna().all(axis=1) all_na = data.stations_data.isna().all(axis=1)
clean_coords = all_na.to_frame().join(clean_coords)[["latitude", "longitude"]] clean_coords = all_na.to_frame().join(clean_coords)[["latitude", "longitude"]]
all_na_coords = clean_coords[all_na] all_na_coords = clean_coords[all_na]
not_na_coords = clean_coords[~all_na] not_na_coords = clean_coords[~all_na]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import matplotlib as mpl import matplotlib as mpl
#definition of plotting function #definition of plotting function
def plot_cells(data, stations, na_stations, discrete=True, plot_stations=False): def plot_cells(data, stations, na_stations, discrete=True, plot_stations=False):
fig = plt.figure(figsize=(9, 18)) fig = plt.figure(figsize=(9, 18))
ax = plt.axes(projection=ccrs.PlateCarree()) ax = plt.axes(projection=ccrs.PlateCarree())
ax.coastlines() ax.coastlines()
gl = ax.gridlines(draw_labels=True) gl = ax.gridlines(draw_labels=True)
gl.top_labels = False gl.top_labels = False
gl.left_labels = False gl.left_labels = False
gl.xlocator = mticker.FixedLocator(data.longitude.values) gl.xlocator = mticker.FixedLocator(data.longitude.values)
gl.ylocator = mticker.FixedLocator(data.latitude.values) gl.ylocator = mticker.FixedLocator(data.latitude.values)
cmap = mpl.cm.viridis cmap = mpl.cm.viridis
if discrete: if discrete:
print(np.unique(data.values)) print(np.unique(data.values))
bounds = np.arange(8) bounds = np.arange(8)
norm = mpl.colors.BoundaryNorm(bounds, cmap.N, extend="both") norm = mpl.colors.BoundaryNorm(bounds, cmap.N, extend="both")
ticks = np.arange(bounds.size + 1)[:-1] + 0.5 ticks = np.arange(bounds.size + 1)[:-1] + 0.5
ticklables = bounds ticklables = bounds
im = plt.pcolormesh( im = plt.pcolormesh(
data.longitude, data.longitude,
data.latitude, data.latitude,
data, data,
transform=ccrs.PlateCarree(), transform=ccrs.PlateCarree(),
cmap=cmap, cmap=cmap,
shading="nearest", shading="nearest",
norm=norm, norm=norm,
) )
cb = fig.colorbar(im, ax=ax, shrink=0.2, aspect=25) cb = fig.colorbar(im, ax=ax, shrink=0.2, aspect=25)
cb.set_ticks(ticks) cb.set_ticks(ticks)
cb.set_ticklabels(ticklables) cb.set_ticklabels(ticklables)
im = plt.pcolormesh( im = plt.pcolormesh(
data.longitude, data.longitude,
data.latitude, data.latitude,
data, data,
transform=ccrs.PlateCarree(), transform=ccrs.PlateCarree(),
cmap=cmap, cmap=cmap,
shading="nearest", shading="nearest",
norm=norm, norm=norm,
) )
else: else:
im = plt.pcolormesh( im = plt.pcolormesh(
data.longitude, data.longitude,
data.latitude, data.latitude,
data, data,
transform=ccrs.PlateCarree(), transform=ccrs.PlateCarree(),
cmap=cmap, cmap=cmap,
shading="nearest", shading="nearest",
) )
cb = fig.colorbar(im, ax=ax, shrink=0.2, aspect=25) cb = fig.colorbar(im, ax=ax, shrink=0.2, aspect=25)
if plot_stations: if plot_stations:
plt.scatter(na_stations["longitude"], na_stations["latitude"], s=1, c="k") plt.scatter(na_stations["longitude"], na_stations["latitude"], s=1, c="k")
plt.scatter(stations["longitude"], stations["latitude"], s=1, c="r") plt.scatter(stations["longitude"], stations["latitude"], s=1, c="r")
plt.tight_layout() plt.tight_layout()
plt.title(f"global ozon at {data.time.values} {data.time.units}") plt.title(f"global ozon at {data.time.values} {data.time.units}")
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#example visualization for two time points #example visualization for two time points
print(not_na_coords) print(not_na_coords)
timestep = 2 timestep = 2
time = ds.time[timestep] time = ds.time[timestep]
data = ds.sel(time=time) data = ds.sel(time=time)
plot_cells(data["mean"], not_na_coords, all_na_coords, discrete=False, plot_stations=True) plot_cells(data["mean"], not_na_coords, all_na_coords, discrete=False, plot_stations=True)
plt.show() plt.show()
plot_cells(data["n"], not_na_coords, all_na_coords, discrete=True) plot_cells(data["n"], not_na_coords, all_na_coords, discrete=True)
plt.show() plt.show()
n_observations = ds["n"].sum(["latitude", "longitude"]) n_observations = ds["n"].sum(["latitude", "longitude"])
plt.plot(ds.time, n_observations) plt.plot(ds.time, n_observations)
print(np.unique(ds["n"])) print(np.unique(ds["n"]))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(data) print(data)
``` ```
......
src/toargridding/contributors.py
+ 3
− 4
View file @ d7699878
...@@ -37,7 +37,7 @@ class contributionsManager: ...@@ -37,7 +37,7 @@ class contributionsManager:
with open(self.contributors_path / f"{self.requestID}.{ext}", "w") as f: with open(self.contributors_path / f"{self.requestID}.{ext}", "w") as f:
for id in self.timeseriesIDs: for id in self.timeseriesIDs:
f.write(f"{id}\n") f.write(f"{id}\n")
return f"curl -d @{self.requestID}.{ext} -X POST {self.endpoint}" return f'curl -X POST "file=@{self.requestID}.{ext}" {self.endpoint}'
def setup_contributors_service(self) -> str: def setup_contributors_service(self) -> str:
# TODO: missing implementation # TODO: missing implementation
raise NotImplementedError("This has not been implemented as this package is not yet operated as a service.") raise NotImplementedError("This has not been implemented as this package is not yet operated as a service.")
...@@ -63,18 +63,17 @@ class contributions_manager_by_name(contributionsManager): ...@@ -63,18 +63,17 @@ class contributions_manager_by_name(contributionsManager):
for name in names: for name in names:
self.timeseriesIDs.add(name) self.timeseriesIDs.add(name)
def id_to_names(self, id : int) -> list[str]: def id_to_names(self, id : int) -> list[str]:
for _ in range(10): for pos in range(10):
req_res = requests.get(f"https://toar-data.fz-juelich.de/api/v2/timeseries/{id}") req_res = requests.get(f"https://toar-data.fz-juelich.de/api/v2/timeseries/{id}")
try: try:
results = req_res.json() results = req_res.json()
break break
except: except:
print("Test debug:", pos, " try for id", id)
sleep(30) sleep(30)
pass pass
else: else:
raise RuntimeError(f"Could not get the response for the timeseries with id {id}.") raise RuntimeError(f"Could not get the response for the timeseries with id {id}.")
names = set() names = set()
for r in results["roles"]: for r in results["roles"]:
try: try:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment