diff --git a/examples/contributor_metadata.ipynb b/examples/contributor_metadata.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..c90845f3a3c2d99f935769a1ca8acfb96dfd2513 --- /dev/null +++ b/examples/contributor_metadata.ipynb @@ -0,0 +1,153 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get Dataset from request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime as dt\n", + "from pathlib import Path\n", + "\n", + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "from toargridding.grids import RegularGrid\n", + "from toargridding.toar_rest_client import (\n", + " AnalysisServiceDownload,\n", + " STATION_LAT,\n", + " STATION_LON,\n", + ")\n", + "from toargridding.metadata import Metadata, TimeSample, AnalysisRequestResult, Coordinates\n", + "from toargridding.variables import Coordinate\n", + "\n", + "from toargridding.contributors import contributionsManager, contributions_manager_by_name\n", + "\n", + "import logging\n", + "from toargridding.defaultLogging import toargridding_defaultLogging\n", + "#setup of logging\n", + "logger = toargridding_defaultLogging()\n", + "logger.addShellLogger(logging.DEBUG)\n", + "logger.logExceptions()\n", + "\n", + "endpoint = \"https://toar-data.fz-juelich.de/api/v2/analysis/statistics/\"\n", + "#starts in directory [path/to/toargridding]/tests\n", + "#maybe adopt the toargridding_base_path for your machine.\n", + "toargridding_base_path = Path(\".\")\n", + "cache_dir = toargridding_base_path / \"cache\"\n", + "data_download_dir = toargridding_base_path / \"results\"\n", + "\n", + "analysis_service = AnalysisServiceDownload(endpoint, cache_dir, data_download_dir, use_downloaded=True)\n", + "my_grid = RegularGrid(1.9, 2.5)\n", + "\n", + "time = TimeSample(dt(2016,1,1), dt(2016,2,28), \"daily\")\n", + "metadata = Metadata.construct(\"mole_fraction_of_ozone_in_air\", time, \"mean\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Contributors as dedicated file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this cell can runs longer than 30minutes\n", + "data = analysis_service.get_data(metadata)\n", + "\n", + "# create contributors endpoint and write result to metadata\n", + "contrib = contributionsManager(metadata.get_id(), data_download_dir)\n", + "contrib.extract_contributors_from_data_frame(data.stations_data)\n", + "metadata.contributors_metadata_field = contrib.setup_contributors_endpoint_for_metadata()\n", + "ds = my_grid.as_xarray(data)\n", + "#store dataset\n", + "ds.to_netcdf(data_download_dir / f\"curl+id-file_{metadata.get_id()}_by_names_inline_{my_grid.get_id()}.nc\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Contributors as single line request" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this cell can runs longer than 30minutes\n", + "data = analysis_service.get_data(metadata)\n", + "\n", + "# create contributors endpoint and write result to metadata\n", + "contrib = contributionsManager(metadata.get_id(), data_download_dir)\n", + "contrib.inline_mode = True\n", + "contrib.extract_contributors_from_data_frame(data.stations_data)\n", + "metadata.contributors_metadata_field = contrib.setup_contributors_endpoint_for_metadata()\n", + "ds = my_grid.as_xarray(data)\n", + "#store dataset\n", + "ds.to_netcdf(data_download_dir / f\"request_in_field_{metadata.get_id()}_by_names_inline_{my_grid.get_id()}.nc\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Contributors by name within field" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# this cell can runs longer than 30minutes\n", + "data = analysis_service.get_data(metadata)\n", + "\n", + "# create contributors endpoint and write result to metadata\n", + "contrib = contributions_manager_by_name(metadata.get_id(), data_download_dir)\n", + "contrib.inline_mode = True\n", + "contrib.extract_contributors_from_data_frame(data.stations_data)\n", + "metadata.contributors_metadata_field = contrib.setup_contributors_endpoint_for_metadata()\n", + "ds = my_grid.as_xarray(data)\n", + "#store dataset\n", + "ds.to_netcdf(data_download_dir / f\"by_name_{metadata.get_id()}_by_names_inline_{my_grid.get_id()}.nc\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "toargridding-g-KQ1Hyq-py3.10", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/toargridding/contributors.py b/src/toargridding/contributors.py index 9fac9f493c92b6101d9bc3854fe8c12fca2bc681..af9b75909e91958876d12117bff4ef3fdcadc4cf 100644 --- a/src/toargridding/contributors.py +++ b/src/toargridding/contributors.py @@ -64,7 +64,7 @@ class contributions_manager_by_name(contributionsManager): self.timeseriesIDs.add(name) def id_to_names(self, id : int) -> list[str]: for pos in range(10): - req_res = requests.get(f"https://toar-data.fz-juelich.de/api/v2/timeseries/{id}") + req_res = requests.get(f"https://toar-data.fz-juelich.de/api/v2/timeseries/{id}", timeout=40) try: results = req_res.json() break @@ -86,5 +86,12 @@ class contributions_manager_by_name(contributionsManager): except: raise RuntimeError("Could not find 'person' or 'organisation' in the response.\n" + str(r)) return names - + def setup_contributors_inline(self) -> str: + return ", ".join([str(id) for id in sorted(self.timeseriesIDs)]) + def setup_contributors_id_file(self) -> str: + ext = "contributors" + with open(self.contributors_path / f"{self.requestID}.{ext}", "w") as f: + for id in self.timeseriesIDs: + f.write(f"{id}\n") + return f'see file {self.requestID}.{ext}"'