From 7487ae48dd740a782e2e730e754113ef0304bcdd Mon Sep 17 00:00:00 2001
From: lukas leufen <l.leufen@fz-juelich.de>
Date: Fri, 24 Apr 2020 13:54:08 +0200
Subject: [PATCH] Added all docstrings, add example for hourly auth into
join_settings. Will become untracked after next commit again.
---
src/configuration/__init__.py | 2 +-
src/configuration/join_settings.py | 18 ++++++++--
src/configuration/path_config.py | 58 +++++++++++++++++++++++++-----
src/helpers/helpers.py | 2 ++
4 files changed, 69 insertions(+), 11 deletions(-)
diff --git a/src/configuration/__init__.py b/src/configuration/__init__.py
index d48bf3ac..4b461174 100644
--- a/src/configuration/__init__.py
+++ b/src/configuration/__init__.py
@@ -1,2 +1,2 @@
-
+"""Collection of configuration functions, paths and classes."""
from .path_config import ROOT_PATH, prepare_host, set_experiment_name, set_bootstrap_path, check_path_and_create
\ No newline at end of file
diff --git a/src/configuration/join_settings.py b/src/configuration/join_settings.py
index a7ee9231..22d8b813 100644
--- a/src/configuration/join_settings.py
+++ b/src/configuration/join_settings.py
@@ -1,10 +1,24 @@
-def join_settings(sampling="daily"):
+"""Settings to access not public join data."""
+from typing import Tuple, Dict
+
+
+def join_settings(sampling="daily") -> Tuple[str, Dict]:
+ """
+ Set url for join and required headers.
+
+ Headers information is not required for daily resolution. For hourly data "Authorization": "<yourtoken>" is required
+ to retrieve any data at all.
+
+ :param sampling: temporal resolution to access. Hourly data requires authorisation.
+
+ :return: Service url and optional headers
+ """
if sampling == "daily": # pragma: no branch
TOAR_SERVICE_URL = 'https://join.fz-juelich.de/services/rest/surfacedata/'
headers = {}
elif sampling == "hourly":
TOAR_SERVICE_URL = 'https://join.fz-juelich.de/services/rest/surfacedata/'
- headers = {}
+ headers = {"Authorization": "Token 12345"}
else:
raise NameError(f"Given sampling {sampling} is not supported, choose from either daily or hourly sampling.")
return TOAR_SERVICE_URL, headers
diff --git a/src/configuration/path_config.py b/src/configuration/path_config.py
index 6b838472..6fdbbd29 100644
--- a/src/configuration/path_config.py
+++ b/src/configuration/path_config.py
@@ -1,12 +1,25 @@
+"""Functions related to path and os name setting."""
import logging
import os
import re
import socket
+from typing import Tuple
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
-def prepare_host(create_new=True, sampling="daily"):
+def prepare_host(create_new=True, sampling="daily") -> str:
+ """
+ Set up host path.
+
+ Warning: This functions can only handle known hosts. For the moment, please add your hostname hardcoded here. For
+ future, this will be replace by a more flexible configuration file setup.
+
+ :param create_new: Create new path if enabled
+ :param sampling: sampling rate to separate data physically by temporal resolution
+
+ :return: full path of data
+ """
hostname = socket.gethostname()
runner_regex = re.compile(r"runner-.*-project-2411-concurrent-\d+")
try:
@@ -41,12 +54,25 @@ def prepare_host(create_new=True, sampling="daily"):
return path
-def set_experiment_name(experiment_date=None, experiment_path=None, sampling=None):
- if experiment_date is None:
+def set_experiment_name(experiment_name=None, experiment_path=None, sampling=None) -> Tuple[str, str]:
+ """
+ Set name of experiment and its path.
+
+ * Experiment name is set to `TestExperiment` if not provided in kwargs. If a name is given, this string is expanded
+ by suffix `_network`. Experiment name is always expanded by `_<sampling>` as ending suffix if sampling is given.
+ * Experiment path is set to `ROOT_PATH/<exp_name>` if not provided or otherwise use `<experiment_path>/<exp_name>`
+
+ :param experiment_name: custom experiment name
+ :param experiment_path: custom experiment path
+ :param sampling: sampling rate as string to add to experiment name
+
+ :return: experiment name and full experiment path
+ """
+ if experiment_name is None:
experiment_name = "TestExperiment"
else:
- experiment_name = f"{experiment_date}_network"
- if sampling == "hourly":
+ experiment_name = f"{experiment_name}_network"
+ if sampling is not None:
experiment_name += f"_{sampling}"
if experiment_path is None:
experiment_path = os.path.abspath(os.path.join(ROOT_PATH, experiment_name))
@@ -55,14 +81,30 @@ def set_experiment_name(experiment_date=None, experiment_path=None, sampling=Non
return experiment_name, experiment_path
-def set_bootstrap_path(bootstrap_path, data_path, sampling):
+def set_bootstrap_path(bootstrap_path: str, data_path: str, sampling: str) -> str:
+ """
+ Set path for bootstrap input data.
+
+ Either use given bootstrap_path or create additional folder in same directory like data path.
+
+ :param bootstrap_path: custom path to store bootstrap data
+ :param data_path: path of data for default bootstrap path
+ :param sampling: sampling rate to add, if path is set to default
+
+ :return: full bootstrap path
+ """
if bootstrap_path is None:
bootstrap_path = os.path.join(data_path, "..", f"bootstrap_{sampling}")
check_path_and_create(bootstrap_path)
- return bootstrap_path
+ return os.path.abspath(bootstrap_path)
+
+def check_path_and_create(path: str) -> None:
+ """
+ Check a given path and create if not existing.
-def check_path_and_create(path):
+ :param path: path to check and create
+ """
try:
os.makedirs(path)
logging.debug(f"Created path: {path}")
diff --git a/src/helpers/helpers.py b/src/helpers/helpers.py
index a6023bd8..968ee538 100644
--- a/src/helpers/helpers.py
+++ b/src/helpers/helpers.py
@@ -29,6 +29,7 @@ def dict_to_xarray(d: Dict, coordinate_name: str) -> xr.DataArray:
:param d: dictionary with 2D-xarrays
:param coordinate_name: name of the new created axis (2D -> 3D)
+
:return: combined xarray
"""
xarray = None
@@ -51,6 +52,7 @@ def float_round(number: float, decimals: int = 0, round_type: Callable = math.ce
:param decimals: numbers of decimals of the rounding operations (default 0 -> round to next integer value)
:param round_type: the actual rounding operation. Can be any callable function like math.ceil, math.floor or python
built-in round operation.
+
:return: rounded number with desired precision
"""
multiplier = 10. ** decimals
--
GitLab