Skip to content
Snippets Groups Projects
Commit d7699878 authored by Carsten Hinz's avatar Carsten Hinz
Browse files

adopted script to test example class.

parent 29e04e74
No related branches found
No related tags found
3 merge requests!11Creation of first beta release version,!10change in metadata of notebook,!9change in metadata of notebook
Hide whitespace changes
Inline Side-by-side
examples/quality_controll.ipynb
+ 3
− 3
View file @ d7699878
%% Cell type:markdown id: tags:
### Get Dataset from request
%% Cell type:code id: tags:
``` python
from datetime import datetime as dt
from pathlib import Path
import pandas as pd
import numpy as np
from toargridding.grids import RegularGrid
from toargridding.toar_rest_client import (
AnalysisServiceDownload,
STATION_LAT,
STATION_LON,
)
from toargridding.metadata import Metadata, TimeSample, AnalysisRequestResult, Coordinates
from toargridding.variables import Coordinate
from toargridding.contributors import contributionsManager
from toargridding.contributors import contributionsManager, contributions_manager_by_name
import logging
from toargridding.defaultLogging import toargridding_defaultLogging
#setup of logging
logger = toargridding_defaultLogging()
logger.addShellLogger(logging.DEBUG)
logger.logExceptions()
endpoint = "https://toar-data.fz-juelich.de/api/v2/analysis/statistics/"
#starts in directory [path/to/toargridding]/tests
#maybe adopt the toargridding_base_path for your machine.
toargridding_base_path = Path(".")
cache_dir = toargridding_base_path / "cache"
data_download_dir = toargridding_base_path / "results"
analysis_service = AnalysisServiceDownload(endpoint, cache_dir, data_download_dir, use_downloaded=True)
my_grid = RegularGrid(1.9, 2.5)
time = TimeSample(dt(2016,1,1), dt(2016,2,28), "daily")
metadata = Metadata.construct("mole_fraction_of_ozone_in_air", time, "mean")
```
%% Cell type:code id: tags:
``` python
# this cell can runs longer than 30minutes
data = analysis_service.get_data(metadata)
# create contributors endpoint and write result to metadata
contrib = contributionsManager(metadata.get_id(), data_download_dir)
contrib = contributions_manager_by_name(metadata.get_id(), data_download_dir)
contrib.extract_contributors_from_data_frame(data.stations_data)
metadata.contributors_metadata_field = contrib.setup_contributors_endpoint_for_metadata()
ds = my_grid.as_xarray(data)
#store dataset
ds.to_netcdf(data_download_dir / f"{metadata.get_id()}_{my_grid.get_id()}.nc")
ds.to_netcdf(data_download_dir / f"{metadata.get_id()}_by_names_inline_{my_grid.get_id()}.nc")
```
%% Cell type:markdown id: tags:
### Visual inspection
%% Cell type:code id: tags:
``` python
#calculation of coordinates for plotting
#especially separation of coordinates with results and without results.
import cartopy.crs as ccrs
import matplotlib.pyplot as plt
import matplotlib.ticker as mticker
mean_data = ds["mean"]
clean_coords = data.stations_coords
all_na = data.stations_data.isna().all(axis=1)
clean_coords = all_na.to_frame().join(clean_coords)[["latitude", "longitude"]]
all_na_coords = clean_coords[all_na]
not_na_coords = clean_coords[~all_na]
```
%% Cell type:code id: tags:
``` python
import matplotlib as mpl
#definition of plotting function
def plot_cells(data, stations, na_stations, discrete=True, plot_stations=False):
fig = plt.figure(figsize=(9, 18))
ax = plt.axes(projection=ccrs.PlateCarree())
ax.coastlines()
gl = ax.gridlines(draw_labels=True)
gl.top_labels = False
gl.left_labels = False
gl.xlocator = mticker.FixedLocator(data.longitude.values)
gl.ylocator = mticker.FixedLocator(data.latitude.values)
cmap = mpl.cm.viridis
if discrete:
print(np.unique(data.values))
bounds = np.arange(8)
norm = mpl.colors.BoundaryNorm(bounds, cmap.N, extend="both")
ticks = np.arange(bounds.size + 1)[:-1] + 0.5
ticklables = bounds
im = plt.pcolormesh(
data.longitude,
data.latitude,
data,
transform=ccrs.PlateCarree(),
cmap=cmap,
shading="nearest",
norm=norm,
)
cb = fig.colorbar(im, ax=ax, shrink=0.2, aspect=25)
cb.set_ticks(ticks)
cb.set_ticklabels(ticklables)
im = plt.pcolormesh(
data.longitude,
data.latitude,
data,
transform=ccrs.PlateCarree(),
cmap=cmap,
shading="nearest",
norm=norm,
)
else:
im = plt.pcolormesh(
data.longitude,
data.latitude,
data,
transform=ccrs.PlateCarree(),
cmap=cmap,
shading="nearest",
)
cb = fig.colorbar(im, ax=ax, shrink=0.2, aspect=25)
if plot_stations:
plt.scatter(na_stations["longitude"], na_stations["latitude"], s=1, c="k")
plt.scatter(stations["longitude"], stations["latitude"], s=1, c="r")
plt.tight_layout()
plt.title(f"global ozon at {data.time.values} {data.time.units}")
```
%% Cell type:code id: tags:
``` python
#example visualization for two time points
print(not_na_coords)
timestep = 2
time = ds.time[timestep]
data = ds.sel(time=time)
plot_cells(data["mean"], not_na_coords, all_na_coords, discrete=False, plot_stations=True)
plt.show()
plot_cells(data["n"], not_na_coords, all_na_coords, discrete=True)
plt.show()
n_observations = ds["n"].sum(["latitude", "longitude"])
plt.plot(ds.time, n_observations)
print(np.unique(ds["n"]))
```
%% Cell type:code id: tags:
``` python
print(data)
```
......
src/toargridding/contributors.py
+ 3
− 4
View file @ d7699878
......@@ -37,7 +37,7 @@ class contributionsManager:
with open(self.contributors_path / f"{self.requestID}.{ext}", "w") as f:
for id in self.timeseriesIDs:
f.write(f"{id}\n")
return f"curl -d @{self.requestID}.{ext} -X POST {self.endpoint}"
return f'curl -X POST "file=@{self.requestID}.{ext}" {self.endpoint}'
def setup_contributors_service(self) -> str:
# TODO: missing implementation
raise NotImplementedError("This has not been implemented as this package is not yet operated as a service.")
......@@ -63,18 +63,17 @@ class contributions_manager_by_name(contributionsManager):
for name in names:
self.timeseriesIDs.add(name)
def id_to_names(self, id : int) -> list[str]:
for _ in range(10):
for pos in range(10):
req_res = requests.get(f"https://toar-data.fz-juelich.de/api/v2/timeseries/{id}")
try:
results = req_res.json()
break
except:
print("Test debug:", pos, " try for id", id)
sleep(30)
pass
else:
raise RuntimeError(f"Could not get the response for the timeseries with id {id}.")
names = set()
for r in results["roles"]:
try:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment