Newer
Older
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"outputs": [
{
"ename": "NameError",
"evalue": "name 'kwargs' is not defined",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[1], line 8\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtoargridding\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtoar_rest_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AnalysisServiceDownload, Connection\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtoargridding\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgrids\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m RegularGrid\n\u001b[0;32m----> 8\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtoargridding\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mgridding\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m get_gridded_toar_data\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtoargridding\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mmetadata\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TimeSample\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtoargridding\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdefaultLogging\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m toargridding_defaultLogging\n",
"File \u001b[0;32m~/Eigene Daten/FZJ/JSC/workingDirectories/TOAR/toargridding/src/toargridding/gridding.py:21\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtoargridding\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtoar_rest_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AnalysisService\n\u001b[1;32m 11\u001b[0m GriddedResult \u001b[38;5;241m=\u001b[39m namedtuple(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mGriddedResult\u001b[39m\u001b[38;5;124m\"\u001b[39m, [\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdataset\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmetadata\u001b[39m\u001b[38;5;124m\"\u001b[39m])\n\u001b[1;32m 14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_gridded_toar_data\u001b[39m(\n\u001b[1;32m 15\u001b[0m analysis_service: AnalysisService,\n\u001b[1;32m 16\u001b[0m grid: GridDefinition,\n\u001b[1;32m 17\u001b[0m time: TimeSample,\n\u001b[1;32m 18\u001b[0m variables: \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mstr\u001b[39m],\n\u001b[1;32m 19\u001b[0m stats: \u001b[38;5;28mlist\u001b[39m[\u001b[38;5;28mstr\u001b[39m],\n\u001b[1;32m 20\u001b[0m contributors_path : Path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m---> 21\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39m\u001b[43mkwargs\u001b[49m,\n\u001b[1;32m 22\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mlist\u001b[39m[xr\u001b[38;5;241m.\u001b[39mDataset], \u001b[38;5;28mlist\u001b[39m[Metadata]]:\n\u001b[1;32m 23\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"API to download data as xarrays\u001b[39;00m\n\u001b[1;32m 24\u001b[0m \n\u001b[1;32m 25\u001b[0m \u001b[38;5;124;03m The function creates all combinations of the variable and stats list\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[38;5;124;03m Gridded datasets for each combination of variables and stats and appropriate metadata for each dataset.\u001b[39;00m\n\u001b[1;32m 43\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m 45\u001b[0m metadatas \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 46\u001b[0m Metadata\u001b[38;5;241m.\u001b[39mconstruct(standard_name\u001b[38;5;241m=\u001b[39mvar, time\u001b[38;5;241m=\u001b[39mtime, stat\u001b[38;5;241m=\u001b[39mstat, moreOptions\u001b[38;5;241m=\u001b[39mkwargs)\n\u001b[1;32m 47\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m var, stat \u001b[38;5;129;01min\u001b[39;00m product(variables, stats)\n\u001b[1;32m 48\u001b[0m ]\n",
"\u001b[0;31mNameError\u001b[0m: name 'kwargs' is not defined"
]
}
],
Carsten Hinz
committed
"import logging\n",
"from datetime import datetime as dt\n",
"from collections import namedtuple\n",
"from pathlib import Path\n",
"\n",
"from toargridding.toar_rest_client import AnalysisServiceDownload, Connection\n",
"from toargridding.grids import RegularGrid\n",
"from toargridding.gridding import get_gridded_toar_data\n",
Carsten Hinz
committed
"from toargridding.metadata import TimeSample\n",
"\n",
"from toargridding.defaultLogging import toargridding_defaultLogging\n",
"\n",
"#setup of logging\n",
Carsten Hinz
committed
"logger = toargridding_defaultLogging()\n",
"logger.addShellLogger(logging.DEBUG)\n",
"logger.logExceptions()\n",
"logger.addRotatingLogFile(Path(\"log/produce_data_withOptional.log\"))#we need to explicitly set a logfile\n",
"#logger.addSysLogger(logging.DEBUG)"
]
},
{
"cell_type": "code",
"Config = namedtuple(\"Config\", [\"grid\", \"time\", \"variables\", \"stats\",\"moreOptions\"])\n",
"\n",
"#moreOptions is implemented as a dict to add additional arguments to the query to the REST API\n",
"#For example the field toar1_category with its possible values Urban, RuralLowElevation, RuralHighElevation and Unclassified can be added.\n",
"#see page 18 in https://toar-data.fz-juelich.de/sphinx/TOAR_UG_Vol03_Database/build/latex/toardatabase--userguide.pdf\n",
"#or type_of_area with urban, suburban and rural on page 20 can be used\n",
"\n",
"details4Query ={\n",
" #\"toar1_category\" : \"Urban\" #uncomment if wished:-)\n",
" #\"toar1_category\" : \"RuralLowElevation\" #uncomment if wished:-)\n",
" #\"toar1_category\" : \"RuralHighElevation\" #uncomment if wished:-)\n",
" #\"type_of_area\" : \"Urban\" #also test Rural, Suburban,\n",
" \"type_of_area\" : \"Rural\" #also test Rural, Suburban,\n",
" #\"type_of_area\" : \"Suburban\" #also test Rural, Suburban,\n",
"}\n",
"grid = RegularGrid( lat_resolution=1.9, lon_resolution=2.5, )\n",
"configs = dict()\n",
" valid_data = Config(\n",
" grid,\n",
" #TimeSample( start=dt(2000+year,1,1), end=dt(2000+year,12,31), sampling=\"daily\"),#possibly adopt range:-)\n",
" TimeSample( start=dt(2000+year,1,1), end=dt(2000+year,12,31), sampling=\"monthly\"),#possibly adopt range:-)\n",
" [\"mole_fraction_of_ozone_in_air\"],#variable name\n",
" #[ \"mean\", \"dma8epax\"],# will start one request after another other...\n",
" #[ \"dma8epa_strict\" ],\n",
" [ \"mean\" ],\n",
" details4Query\n",
" )\n",
" \n",
]
},
{
"cell_type": "code",
"#CAVE: this cell runs about 45minutes per requested year. therefore we increase the waiting duration to 1h per request.\n",
"#the processing is done on the server of the TOAR database.\n",
"#a restart of the cell continues the request to the REST API if the requested data are ready for download\n",
"# The download can also take a few minutes\n",
"\n",
"stats_endpoint = \"https://toar-data.fz-juelich.de/api/v2/analysis/statistics/\"\n",
"cache_basepath = Path(\"cache\")\n",
"result_basepath = Path(\"results\")\n",
"cache_basepath.mkdir(exist_ok=True)\n",
"analysis_service = AnalysisServiceDownload(stats_endpoint=stats_endpoint, cache_dir=cache_basepath, sample_dir=result_basepath, use_downloaded=True)\n",
"\n",
"# maybe adopt the interval for requesting the results and the total duration, before the client pauses the requests.\n",
"# as the requests take about 45min, it is more suitable to wait 60min before timing out the requests than the original 30min.\n",
"analysis_service.connection.setRequestTimes(interval_min=5, maxWait_min=60)\n",
"for person, config in configs.items():\n",
" print(f\"\\nProcessing {person}:\")\n",
" print(f\"--------------------\")\n",
" datasets, metadatas = get_gridded_toar_data(\n",
" analysis_service=analysis_service,\n",
" grid=config.grid,\n",
" time=config.time,\n",
" variables=config.variables,\n",
" contributors_path=result_basepath\n",
" )\n",
"\n",
" for dataset, metadata in zip(datasets, metadatas):\n",
" dataset.to_netcdf(result_basepath / f\"{metadata.get_id()}_{config.grid.get_id()}.nc\")\n",
]
}
],
"metadata": {
"kernelspec": {
"display_name": "toargridding-8RVrxzmn-py3.11",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}