Skip to content
Snippets Groups Projects
Commit 4ffb095e authored by Carsten Hinz's avatar Carsten Hinz
Browse files

added different keyword for station category 

enabled execution with additional query parameters
parent 206a6d9e
No related branches found
No related tags found
1 merge request!11Creation of first beta release version
Hide whitespace changes
Inline Side-by-side
tests/produce_data.ipynb
+ 49
− 9
View file @ 4ffb095e
%% Cell type:code id: tags:
``` python
from datetime import datetime as dt
from collections import namedtuple
from pathlib import Path
from toargridding.toar_rest_client import AnalysisServiceDownload
from toargridding.grids import RegularGrid
from toargridding.gridding import get_gridded_toar_data
from toargridding.metadata import TimeSample
```
%% Cell type:code id: tags:
``` python
#creation of request.
Config = namedtuple("Config", ["grid", "time", "variables", "stats","moreOptions"])
#moreOptions is implemented as a dict to add additional arguments to the query to the REST API
#For example the field toar1_category with its possible values Urban, RuralLowElevation, RuralHighElevation and Unclassified can be added.
#see page 18 in https://toar-data.fz-juelich.de/sphinx/TOAR_UG_Vol03_Database/build/latex/toardatabase--userguide.pdf
details4Query ={
#"toar1_category" : "Urban" #uncomment if wished:-)
#"toar1_category" : "RuralLowElevation" #uncomment if wished:-)
#"toar1_category" : "RuralHighElevation" #uncomment if wished:-)
}
valid_data = Config(
RegularGrid( lat_resolution=1.9, lon_resolution=2.5, ),
TimeSample( start=dt(2000,1,1), end=dt(2019,12,31), sampling="daily"),#possibly adopt range:-)
["mole_fraction_of_ozone_in_air"],#variable name
#[ "mean", "dma8epax"],# will start one request after another other...
[ "dma8epax", "mean" ],# will start one request after another other...
details4Query
)
configs = {
"test_ta" : valid_data
}
#testing access:
config = configs["test_ta"]
config.grid
```
%% Output
<toargridding.grids.RegularGrid at 0x7f024eb0a5d0>
%% Cell type:code id: tags:
``` python
#CAVE: this cell runs about 30minutes per requested year
#the processing is done on the server of the TOAR database.
#a restart of the cell continues the request to the REST API if the requested data are ready for download
# The download can also take a few minutes
stats_endpoint = "https://toar-data.fz-juelich.de/api/v2/analysis/statistics/"
cache_basepath = Path("cache")
result_basepath = Path("results")
cache_basepath.mkdir(exist_ok=True)
result_basepath.mkdir(exist_ok=True)
analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath)
analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath, use_downloaded=True)
for person, config in configs.items():
datasets, metadatas = get_gridded_toar_data(
analysis_service=analysis_service,
grid=config.grid,
time=config.time,
variables=config.variables,
stats=config.stats
#**config.moreOptions
stats=config.stats,
**config.moreOptions
)
for dataset, metadata in zip(datasets, metadatas):
dataset.to_netcdf(result_basepath / f"{metadata.get_id()}.nc")
print(metadata.get_id())
```
%% Output
Performing request to TOAR DB
load status endpoint from cache
try: 1, wait_time: 300
try: 2, wait_time: 300
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
Cell In[24], line 14
11 analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath, use_downloaded=True)
13 for person, config in configs.items():
---> 14 datasets, metadatas = get_gridded_toar_data(
15 analysis_service=analysis_service,
16 grid=config.grid,
17 time=config.time,
18 variables=config.variables,
19 stats=config.stats,
20 **config.moreOptions
21 )
23 for dataset, metadata in zip(datasets, metadatas):
24 dataset.to_netcdf(result_basepath / f"{metadata.get_id()}.nc")
File ~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/gridding.py:49, in get_gridded_toar_data(analysis_service, grid, time, variables, stats, **kwargs)
47 datasets = []
48 for metadata in metadatas: # standart_name ?
---> 49 data = analysis_service.get_data(metadata)
50 ds = grid.as_xarray(data)
51 datasets.append(ds)
File ~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:307, in AnalysisService.get_data(self, metadata)
293 def get_data(self, metadata: Metadata) -> AnalysisRequestResult:
294 """main function to obtain data from the TOAR DB
295
296 Handles requesting and loading of data into memory as soon as they are available.
(...)
304 Requested data and statistics, station coordinates and metadata of the request
305 """
--> 307 timeseries, timeseries_metadata = self.get_timeseries_and_metadata(metadata)
308 coords = self.get_clean_coords(timeseries_metadata)
309 timeseries = self.get_clean_timeseries(timeseries, metadata)
File ~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:447, in AnalysisServiceDownload.get_timeseries_and_metadata(self, metadata)
445 if needs_fresh_download:
446 print("Performing request to TOAR DB")
--> 447 response = self.connection.get(query_options)
448 with open(filename, "w+b") as downloaded_file:
449 downloaded_file.write(response.content)
File ~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:189, in Connection.get(self, query_options)
187 for i, wait_time in enumerate(self.wait_seconds):
188 print(f"try: {i+1}, wait_time: {wait_time}")
--> 189 response = self.wait_and_get(status_endpoint, wait_secs=wait_time)
190 if response.headers["Content-Type"] == "application/zip":
191 return response
File ~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/toargridding/toar_rest_client.py:273, in wait_and_get(self, endpoint, query_options, wait_secs, timeout)
257 def wait_and_get(
258 self, endpoint : str, query_options : Dict =None, wait_secs=None, timeout=(3.05, 20)
259 ):
260 """accesses given endpoint
261
262 Parameters:
(...)
271 timeout for the request.
272 """
--> 273 if wait_secs:
274 time.sleep(wait_secs)
276 return requests.get(endpoint, params=query_options, timeout=timeout)
KeyboardInterrupt:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment