Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime as dt\n",
"from collections import namedtuple\n",
"from pathlib import Path\n",
"\n",
"from toargridding.toar_rest_client import AnalysisServiceDownload, Connection\n",
"from toargridding.grids import RegularGrid\n",
"from toargridding.gridding import get_gridded_toar_data\n",
"from toargridding.metadata import TimeSample\n",
"\n",
"from toargridding.metadata_utilities import countryCodes"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#creation of request.\n",
"\n",
"Config = namedtuple(\"Config\", [\"grid\", \"time\", \"variables\", \"stats\",\"moreOptions\"])\n",
"\n",
"#see page 18 in https://toar-data.fz-juelich.de/sphinx/TOAR_UG_Vol03_Database/build/latex/toardatabase--userguide.pdf\n",
"\n",
"varName = \"country\"\n",
"stationCountries = countryCodes()\n",
"validCountries = stationCountries.getValidVocabular(controlName=\"Country Code\", varName=varName)\n",
"\n",
"grid = RegularGrid( lat_resolution=1.9, lon_resolution=2.5, )\n",
"\n",
"configs = dict()\n",
"for country in validCountries:\n",
" valid_data = Config(\n",
" grid,\n",
" TimeSample( start=dt(2000,1,1), end=dt(2018,12,31), sampling=\"daily\"),#possibly adopt range:-)\n",
" [\"mole_fraction_of_ozone_in_air\"],#variable name\n",
" [ \"dma8epa_strict\" ],\n",
" {varName : country}\n",
" )\n",
" \n",
" configs[f\"test_ta{country}\"] = valid_data\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#CAVE: this cell runs about 45minutes per requested year. therefore we increase the waiting duration to 1h per request.\n",
"#the processing is done on the server of the TOAR database.\n",
"#a restart of the cell continues the request to the REST API if the requested data are ready for download\n",
"# The download can also take a few minutes\n",
"\n",
"stats_endpoint = \"https://toar-data.fz-juelich.de/api/v2/analysis/statistics/\"\n",
"cache_basepath = Path(\"cache\")\n",
"result_basepath = Path(\"results\")\n",
"cache_basepath.mkdir(exist_ok=True)\n",
"result_basepath.mkdir(exist_ok=True)\n",
"analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath, use_downloaded=True)\n",
"\n",
"Connection.DEBUG=True\n",
"\n",
"# maybe adopt the interval for requesting the results and the total duration, before the client pauses the requests.\n",
"# as the requests take about 45min, it is more suitable to wait 60min before timing out the requests than the original 30min.\n",
"analysis_service.connection.setRequestTimes(interval_min=5, maxWait_min=60)\n",
"\n",
"createdFiles = []\n",
"\n",
"for person, config in configs.items():\n",
" print(f\"\\nProcessing {person}:\")\n",
" print(f\"--------------------\")\n",
" try:\n",
" datasets, metadatas = get_gridded_toar_data(\n",
" analysis_service=analysis_service,\n",
" grid=config.grid,\n",
" time=config.time,\n",
" variables=config.variables,\n",
" stats=config.stats,\n",
" **config.moreOptions\n",
" )\n",
" except KeyError as e:\n",
" print(\"failed for \", person)\n",
" continue\n",
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
"\n",
" for dataset, metadata in zip(datasets, metadatas):\n",
" outName = result_basepath / f\"{metadata.get_id()}_{config.grid.get_id()}.nc\"\n",
" dataset.to_netcdf(outName)\n",
" createdFiles.append(outName)\n",
" print(metadata.get_id())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"##TODO: now we only need to combine all the obtained results...\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "toargridding-8RVrxzmn-py3.11",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
}
},
"nbformat": 4,
"nbformat_minor": 2
}