Newer
Older
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime as dt\n",
"from collections import namedtuple\n",
"from pathlib import Path\n",
"\n",
Carsten Hinz
committed
"from toargridding.toar_rest_client import AnalysisServiceDownload, Connection, EmptyDataError\n",
"from toargridding.grids import RegularGrid\n",
"from toargridding.gridding import get_gridded_toar_data\n",
"from toargridding.metadata import TimeSample\n",
"\n",
"from toargridding.metadata_utilities import countryCodes\n",
"\n",
"from toargridding.defaultLogging import toargridding_defaultLogging\n",
"#setup of logging\n",
"logger = toargridding_defaultLogging()\n",
"logger.addShellLogger(logging.DEBUG)\n",
"logger.logExceptions()\n",
"logger.addRotatingLogFile_scriptName(__file__)\n",
"logger.addRotatingLogFile(Path(\"log/produce_data_withOptional_country.log\"))#we need to explicitly set a logfile\n",
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#creation of request.\n",
"\n",
"Config = namedtuple(\"Config\", [\"grid\", \"time\", \"variables\", \"stats\",\"moreOptions\"])\n",
"\n",
"#see page 18 in https://toar-data.fz-juelich.de/sphinx/TOAR_UG_Vol03_Database/build/latex/toardatabase--userguide.pdf\n",
"\n",
"varName = \"country\"\n",
"stationCountries = countryCodes()\n",
"validCountries = stationCountries.getValidVocabular(controlName=\"Country Code\", varName=varName)\n",
"\n",
"grid = RegularGrid( lat_resolution=1.9, lon_resolution=2.5, )\n",
"\n",
"configs = dict()\n",
"for country in validCountries:\n",
" valid_data = Config(\n",
" grid,\n",
" TimeSample( start=dt(2000,1,1), end=dt(2018,12,31), sampling=\"daily\"),#possibly adopt range:-)\n",
" [\"mole_fraction_of_ozone_in_air\"],#variable name\n",
" [ \"dma8epa_strict\" ],\n",
" {varName : country}\n",
" )\n",
" \n",
" configs[f\"test_ta{country}\"] = valid_data\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#CAVE: this cell runs about 45minutes per requested year. therefore we increase the waiting duration to 1h per request.\n",
"#the processing is done on the server of the TOAR database.\n",
"#a restart of the cell continues the request to the REST API if the requested data are ready for download\n",
"# The download can also take a few minutes\n",
"\n",
"stats_endpoint = \"https://toar-data.fz-juelich.de/api/v2/analysis/statistics/\"\n",
"cache_basepath = Path(\"cache\")\n",
"result_basepath = Path(\"results\")\n",
"cache_basepath.mkdir(exist_ok=True)\n",
"result_basepath.mkdir(exist_ok=True)\n",
"analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath, use_downloaded=True)\n",
"\n",
"\n",
"# maybe adopt the interval for requesting the results and the total duration, before the client pauses the requests.\n",
"# as the requests take about 45min, it is more suitable to wait 60min before timing out the requests than the original 30min.\n",
"analysis_service.connection.setRequestTimes(interval_min=5, maxWait_min=60)\n",
"\n",
"createdFiles = []\n",
"\n",
"for person, config in configs.items():\n",
" print(f\"\\nProcessing {person}:\")\n",
" print(f\"--------------------\")\n",
" try:\n",
" datasets, metadatas = get_gridded_toar_data(\n",
" analysis_service=analysis_service,\n",
" grid=config.grid,\n",
" time=config.time,\n",
" variables=config.variables,\n",
" stats=config.stats,\n",
" **config.moreOptions\n",
" )\n",
Carsten Hinz
committed
" except EmptyDataError as e:\n",
" print(\"failed for \", person)\n",
" continue\n",
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
"\n",
" for dataset, metadata in zip(datasets, metadatas):\n",
" outName = result_basepath / f\"{metadata.get_id()}_{config.grid.get_id()}.nc\"\n",
" dataset.to_netcdf(outName)\n",
" createdFiles.append(outName)\n",
" print(metadata.get_id())"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"##TODO: now we only need to combine all the obtained results...\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "toargridding-8RVrxzmn-py3.11",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
}
},
"nbformat": 4,
"nbformat_minor": 2
}