diff --git a/.coveragerc b/.coveragerc
index baa10de454893675aeedc6275e2c6725b0b84966..69c1dcd3f1ca5068733a54fdb231bab80170169d 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -23,3 +23,6 @@ exclude_lines =
# Don't complain about import statements
import
+
+ # Don't complain about abstract class declarations and placeholders
+ pass
diff --git a/src/datastore.py b/src/datastore.py
new file mode 100644
index 0000000000000000000000000000000000000000..348f28549e716a30605ff2be5894fd5b03b4dc95
--- /dev/null
+++ b/src/datastore.py
@@ -0,0 +1,331 @@
+__author__ = 'Lukas Leufen'
+__date__ = '2019-11-22'
+
+
+from typing import Any, List, Tuple
+
+from abc import ABC
+
+
+class NameNotFoundInDataStore(Exception):
+ """
+ Exception that get raised if given name is not found in the entire data store.
+ """
+ pass
+
+
+class NameNotFoundInScope(Exception):
+ """
+ Exception that get raised if given name is not found in the provided scope, but can be found in other scopes.
+ """
+ pass
+
+
+class EmptyScope(Exception):
+ """
+ Exception that get raised if given scope is not part of the data store.
+ """
+ pass
+
+
+class AbstractDataStore(ABC):
+
+ """
+ Data store for all settings for the experiment workflow to save experiment parameters for the proceeding modules
+ and predefine parameters loaded during the experiment setup phase. The data store is hierarchically structured, so
+ that global settings can be overwritten by local adjustments.
+ """
+ def __init__(self):
+ # empty initialise the data-store variables
+ self._store = {}
+
+ def put(self, name: str, obj: Any, scope: str) -> None:
+ """
+ Abstract method to add an object to the data store
+ :param name: Name of object to store
+ :param obj: The object itself to be stored
+ :param scope: the scope / context of the object, under that the object is valid
+ """
+ pass
+
+ def get(self, name: str, scope: str) -> None:
+ """
+ Abstract method to get an object from the data store
+ :param name: Name to look for
+ :param scope: scope to search the name for
+ :return: the stored object
+ """
+ pass
+
+ def search_name(self, name: str) -> None:
+ """
+ Abstract method to search for all occurrences of given `name` in the entire data store.
+ :param name: Name to look for
+ :return: search result
+ """
+ pass
+
+ def search_scope(self, scope: str) -> None:
+ """
+ Abstract method to search for all object names that are stored for given scope
+ :param scope: scope to look for
+ :return: search result
+ """
+ pass
+
+ def list_all_scopes(self) -> None:
+ """
+ Abstract method to list all scopes in data store
+ :return: all found scopes
+ """
+ pass
+
+
+class DataStoreByVariable(AbstractDataStore):
+
+ """
+ Data store for all settings for the experiment workflow to save experiment parameters for the proceeding modules
+ and predefine parameters loaded during the experiment setup phase. The data store is hierarchically structured, so
+ that global settings can be overwritten by local adjustments.
+
+ This implementation stores data as
+ <variable1>
+ <scope1>: value
+ <scope2>: value
+ <variable2>
+ <scope1>: value
+ <scope3>: value
+ """
+
+ def put(self, name: str, obj: Any, scope: str) -> None:
+ """
+ Store an object `obj` with given `name` under `scope`. In the current implementation, existing entries are
+ overwritten.
+ :param name: Name of object to store
+ :param obj: The object itself to be stored
+ :param scope: the scope / context of the object, under that the object is valid
+ """
+ # open new variable related store with `name` as key if not existing
+ if name not in self._store.keys():
+ self._store[name] = {}
+ self._store[name][scope] = obj
+
+ def get(self, name: str, scope: str) -> Any:
+ """
+ Retrieve an object with `name` from `scope`. If no object can be found in the exact scope, take an iterative
+ look on the levels above. Raises a NameNotFoundInDataStore error, if no object with given name can be found in
+ the entire data store. Raises a NameNotFoundInScope error, if the object is in the data store but not in the
+ given scope and its levels above (could be either included in another scope or a more detailed sub-scope).
+ :param name: Name to look for
+ :param scope: scope to search the name for
+ :return: the stored object
+ """
+ return self._stride_through_scopes(name, scope)[2]
+
+ def _stride_through_scopes(self, name, scope, depth=0):
+ if depth <= scope.count("."):
+ local_scope = scope.rsplit(".", maxsplit=depth)[0]
+ try:
+ return name, local_scope, self._store[name][local_scope]
+ except KeyError:
+ return self._stride_through_scopes(name, scope, depth + 1)
+ else:
+ occurrences = self.search_name(name)
+ if len(occurrences) == 0:
+ raise NameNotFoundInDataStore(f"Couldn't find {name} in data store")
+ else:
+ raise NameNotFoundInScope(f"Couldn't find {name} in scope {scope} . {name} is only defined in "
+ f"{occurrences}")
+
+ def search_name(self, name: str) -> List[str]:
+ """
+ Search for all occurrences of given `name` in the entire data store.
+ :param name: Name to look for
+ :return: list with all scopes and sub-scopes containing an object stored as `name`
+ """
+ return sorted(self._store[name] if name in self._store.keys() else [])
+
+ def search_scope(self, scope: str, current_scope_only=True, return_all=False) -> List[str or Tuple]:
+ """
+ Search for given `scope` and list all object names stored under this scope. To look also for all superior scopes
+ set `current_scope_only=False`. To return the scope and the object's value too, set `return_all=True`.
+ :param scope: scope to look for
+ :param current_scope_only: look only for all names for given scope if true, else search for names from superior
+ scopes too.
+ :param return_all: return name, definition scope and value if True, else just the name
+ :return: list with all object names (if `return_all=False`) or list with tuple of object name, object scope and
+ object value ordered by name (if `return_all=True`)
+ """
+ if current_scope_only:
+ names = []
+ for (k, v) in self._store.items():
+ if scope in v.keys():
+ names.append(k)
+ if len(names) > 0:
+ if return_all:
+ return sorted([(name, scope, self._store[name][scope]) for name in names], key=lambda tup: tup[0])
+ else:
+ return sorted(names)
+ else:
+ raise EmptyScope(f"Given scope {scope} is not part of the data store. Available scopes are: "
+ f"{self.list_all_scopes()}")
+ else:
+ results = []
+ for name in self.list_all_names():
+ try:
+ res = self._stride_through_scopes(name, scope)
+ if return_all:
+ results.append(res)
+ else:
+ results.append(res[0])
+ except (NameNotFoundInDataStore, NameNotFoundInScope):
+ pass
+ if return_all:
+ return sorted(results, key=lambda tup: tup[0])
+ else:
+ return sorted(results)
+
+ def list_all_scopes(self) -> List[str]:
+ """
+ List all available scopes in data store
+ :return: names of all stored objects
+ """
+ scopes = []
+ for v in self._store.values():
+ for scope in v.keys():
+ if scope not in scopes:
+ scopes.append(scope)
+ return sorted(scopes)
+
+ def list_all_names(self) -> List[str]:
+ """
+ List all names available in the data store.
+ :return: all names
+ """
+ return sorted(self._store.keys())
+
+
+class DataStoreByScope(AbstractDataStore):
+
+ """
+ Data store for all settings for the experiment workflow to save experiment parameters for the proceeding modules
+ and predefine parameters loaded during the experiment setup phase. The data store is hierarchically structured, so
+ that global settings can be overwritten by local adjustments.
+
+ This implementation stores data as
+ <scope1>
+ <variable1>: value
+ <variable2>: value
+ <scope2>
+ <variable1>: value
+ <variable3>: value
+ """
+
+ def put(self, name: str, obj: Any, scope: str) -> None:
+ """
+ Store an object `obj` with given `name` under `scope`. In the current implementation, existing entries are
+ overwritten.
+ :param name: Name of object to store
+ :param obj: The object itself to be stored
+ :param scope: the scope / context of the object, under that the object is valid
+ """
+ if scope not in self._store.keys():
+ self._store[scope] = {}
+ self._store[scope][name] = obj
+
+ def get(self, name: str, scope: str) -> Any:
+ """
+ Retrieve an object with `name` from `scope`. If no object can be found in the exact scope, take an iterative
+ look on the levels above. Raises a NameNotFoundInDataStore error, if no object with given name can be found in
+ the entire data store. Raises a NameNotFoundInScope error, if the object is in the data store but not in the
+ given scope and its levels above (could be either included in another scope or a more detailed sub-scope).
+ :param name: Name to look for
+ :param scope: scope to search the name for
+ :return: the stored object
+ """
+ return self._stride_through_scopes(name, scope)[2]
+
+ def _stride_through_scopes(self, name, scope, depth=0):
+ if depth <= scope.count("."):
+ local_scope = scope.rsplit(".", maxsplit=depth)[0]
+ try:
+ return name, local_scope, self._store[local_scope][name]
+ except KeyError:
+ return self._stride_through_scopes(name, scope, depth + 1)
+ else:
+ occurrences = self.search_name(name)
+ if len(occurrences) == 0:
+ raise NameNotFoundInDataStore(f"Couldn't find {name} in data store")
+ else:
+ raise NameNotFoundInScope(f"Couldn't find {name} in scope {scope} . {name} is only defined in "
+ f"{occurrences}")
+
+ def search_name(self, name: str) -> List[str]:
+ """
+ Search for all occurrences of given `name` in the entire data store.
+ :param name: Name to look for
+ :return: list with all scopes and sub-scopes containing an object stored as `name`
+ """
+ keys = []
+ for (key, val) in self._store.items():
+ if name in val.keys():
+ keys.append(key)
+ return sorted(keys)
+
+ def search_scope(self, scope: str, current_scope_only: bool = True, return_all: bool = False) -> List[str or Tuple]:
+ """
+ Search for given `scope` and list all object names stored under this scope. To look also for all superior scopes
+ set `current_scope_only=False`. To return the scope and the object's value too, set `return_all=True`.
+ :param scope: scope to look for
+ :param current_scope_only: look only for all names for given scope if true, else search for names from superior
+ scopes too.
+ :param return_all: return name, definition scope and value if True, else just the name
+ :return: list with all object names (if `return_all=False`) or list with tuple of object name, object scope and
+ object value ordered by name (if `return_all=True`)
+ """
+ if current_scope_only:
+ try:
+ if return_all:
+ return [(name, scope, self._store[scope][name]) for name in sorted(self._store[scope].keys())]
+ else:
+ return sorted(self._store[scope].keys())
+ except KeyError:
+ raise EmptyScope(f"Given scope {scope} is not part of the data store. Available scopes are: "
+ f"{self.list_all_scopes()}")
+ else:
+ results = []
+ for name in self.list_all_names():
+ try:
+ res = self._stride_through_scopes(name, scope)
+ if return_all:
+ results.append(res)
+ else:
+ results.append(res[0])
+ except (NameNotFoundInDataStore, NameNotFoundInScope):
+ pass
+ if return_all:
+ return sorted(results, key=lambda tup: tup[0])
+ else:
+ return sorted(results)
+
+ def list_all_scopes(self) -> List[str]:
+ """
+ List all available scopes in data store
+ :return: names of all stored objects
+ """
+ return sorted(self._store.keys())
+
+ def list_all_names(self) -> List[str]:
+ """
+ List all names available in the data store.
+ :return: all names
+ """
+ names = []
+ scopes = self.list_all_scopes()
+ for scope in scopes:
+ for name in self._store[scope].keys():
+ if name not in names:
+ names.append(name)
+ return sorted(names)
+
+
diff --git a/test/test_datastore.py b/test/test_datastore.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ba76bf8fc9c21553723cba3b2125be2d758e23b
--- /dev/null
+++ b/test/test_datastore.py
@@ -0,0 +1,222 @@
+__author__ = 'Lukas Leufen'
+__date__ = '2019-11-22'
+
+
+from src.datastore import AbstractDataStore, DataStoreByVariable, DataStoreByScope
+from src.datastore import NameNotFoundInDataStore, NameNotFoundInScope, EmptyScope
+import pytest
+
+
+class TestAbstractDataStore:
+
+ @pytest.fixture
+ def ds(self):
+ return AbstractDataStore()
+
+ def test_init(self, ds):
+ assert ds._store == {}
+
+
+class TestDataStoreByVariable:
+
+ @pytest.fixture
+ def ds(self):
+ return DataStoreByVariable()
+
+ def test_put(self, ds):
+ ds.put("number", 3, "general.subscope")
+ assert ds._store["number"]["general.subscope"] == 3
+
+ def test_get(self, ds):
+ ds.put("number", 3, "general.subscope")
+ assert ds.get("number", "general.subscope") == 3
+
+ def test_get_with_sub_scope(self, ds):
+ ds.put("number", 3, "general")
+ ds.put("number", 10, "general.subscope")
+ assert ds.get("number", "general.subscope") == 10
+ assert ds.get("number", "general") == 3
+
+ def test_get_with_not_existing_sub_scope(self, ds):
+ ds.put("number", 3, "general")
+ ds.put("number2", 10, "general.subscope")
+ ds.put("number2", 1, "general")
+ assert ds.get("number", "general.subscope") == 3
+
+ def test_raise_not_in_data_store(self, ds):
+ ds.put("number", 22, "general")
+ with pytest.raises(NameNotFoundInDataStore) as e:
+ ds.get("number3", "general")
+ assert "Couldn't find number3 in data store" in e.value.args[0]
+
+ def test_search(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 22, "general2")
+ ds.put("number", 22, "general.sub")
+ assert ds.search_name("number") == ["general", "general.sub", "general2"]
+
+ def test_raise_not_in_scope(self, ds):
+ ds.put("number", 11, "general.sub")
+ with pytest.raises(NameNotFoundInScope) as e:
+ ds.get("number", "general.sub2")
+ assert "Couldn't find number in scope general.sub2 . number is only defined in ['general.sub']" in e.value.args[0]
+
+ def test_list_all_scopes(self, ds):
+ ds.put("number", 22, "general2")
+ ds.put("number", 11, "general.sub")
+ ds.put("number2", 2, "general.sub")
+ ds.put("number", 3, "general.sub3")
+ ds.put("number", 1, "general")
+ assert ds.list_all_scopes() == ['general', 'general.sub', 'general.sub3', 'general2']
+
+ def test_search_scope(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ assert ds.search_scope("general.sub") == ["number", "number1"]
+
+ def test_search_empty_scope(self, ds):
+ ds.put("number", 22, "general2")
+ ds.put("number", 11, "general.sub")
+ with pytest.raises(EmptyScope) as e:
+ ds.search_scope("general.sub2")
+ assert "Given scope general.sub2 is not part of the data store." in e.value.args[0]
+ assert "Available scopes are: ['general.sub', 'general2']" in e.value.args[0]
+
+ def test_list_all_names(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ assert ds.list_all_names() == ["number", "number1", "number2"]
+
+ def test_search_scope_and_all_superiors(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ assert ds.search_scope("general.sub", current_scope_only=False) == ["number", "number1"]
+ assert ds.search_scope("general.sub.sub", current_scope_only=False) == ["number", "number1", "number2"]
+
+ def test_search_scope_return_all(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ assert ds.search_scope("general.sub", return_all=True) == [("number", "general.sub", 11),
+ ("number1", "general.sub", 22)]
+
+ def test_search_scope_and_all_superiors_return_all(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ ds.put("number", "ABC", "general.sub.sub")
+ assert ds.search_scope("general.sub", current_scope_only=False, return_all=True) == \
+ [("number", "general.sub", 11), ("number1", "general.sub", 22)]
+ assert ds.search_scope("general.sub.sub", current_scope_only=False, return_all=True) == \
+ [("number", "general.sub.sub", "ABC"), ("number1", "general.sub", 22), ("number2", "general.sub.sub", 3)]
+
+
+class TestDataStoreByScope:
+
+ @pytest.fixture
+ def ds(self):
+ return DataStoreByScope()
+
+ def test_put_with_scope(self, ds):
+ ds.put("number", 3, "general.subscope")
+ assert ds._store["general.subscope"]["number"] == 3
+
+ def test_get(self, ds):
+ ds.put("number", 3, "general.subscope")
+ assert ds.get("number", "general.subscope") == 3
+
+ def test_get_with_sub_scope(self, ds):
+ ds.put("number", 3, "general")
+ ds.put("number", 10, "general.subscope")
+ assert ds.get("number", "general.subscope") == 10
+ assert ds.get("number", "general") == 3
+
+ def test_get_with_not_existing_sub_scope(self, ds):
+ ds.put("number", 3, "general")
+ ds.put("number2", 10, "general.subscope")
+ ds.put("number2", 1, "general")
+ assert ds.get("number", "general.subscope") == 3
+
+ def test_raise_not_in_data_store(self, ds):
+ ds.put("number", 22, "general")
+ with pytest.raises(NameNotFoundInDataStore) as e:
+ ds.get("number3", "general")
+ assert "Couldn't find number3 in data store" in e.value.args[0]
+
+ def test_search(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 22, "general2")
+ ds.put("number", 22, "general.sub")
+ assert ds.search_name("number") == ["general", "general.sub", "general2"]
+
+ def test_raise_not_in_scope(self, ds):
+ ds.put("number", 11, "general.sub")
+ with pytest.raises(NameNotFoundInScope) as e:
+ ds.get("number", "general.sub2")
+ assert "Couldn't find number in scope general.sub2 . number is only defined in ['general.sub']" in e.value.args[0]
+
+ def test_list_all_scopes(self, ds):
+ ds.put("number", 22, "general2")
+ ds.put("number", 11, "general.sub")
+ ds.put("number2", 2, "general.sub")
+ ds.put("number", 3, "general.sub3")
+ ds.put("number", 1, "general")
+ assert ds.list_all_scopes() == ['general', 'general.sub', 'general.sub3', 'general2']
+
+ def test_search_scope(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ assert ds.search_scope("general.sub") == ["number", "number1"]
+
+ def test_search_empty_scope(self, ds):
+ ds.put("number", 22, "general2")
+ ds.put("number", 11, "general.sub")
+ with pytest.raises(EmptyScope) as e:
+ ds.search_scope("general.sub2")
+ assert "Given scope general.sub2 is not part of the data store." in e.value.args[0]
+ assert "Available scopes are: ['general.sub', 'general2']" in e.value.args[0]
+
+ def test_list_all_names(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ assert ds.list_all_names() == ["number", "number1", "number2"]
+
+ def test_search_scope_and_all_superiors(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ assert ds.search_scope("general.sub", current_scope_only=False) == ["number", "number1"]
+ assert ds.search_scope("general.sub.sub", current_scope_only=False) == ["number", "number1", "number2"]
+
+ def test_search_scope_return_all(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ assert ds.search_scope("general.sub", return_all=True) == [("number", "general.sub", 11),
+ ("number1", "general.sub", 22)]
+
+ def test_search_scope_and_all_superiors_return_all(self, ds):
+ ds.put("number", 22, "general")
+ ds.put("number", 11, "general.sub")
+ ds.put("number1", 22, "general.sub")
+ ds.put("number2", 3, "general.sub.sub")
+ ds.put("number", "ABC", "general.sub.sub")
+ assert ds.search_scope("general.sub", current_scope_only=False, return_all=True) == \
+ [("number", "general.sub", 11), ("number1", "general.sub", 22)]
+ assert ds.search_scope("general.sub.sub", current_scope_only=False, return_all=True) == \
+ [("number", "general.sub.sub", "ABC"), ("number1", "general.sub", 22), ("number2", "general.sub.sub", 3)]
+