Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
MLAir
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
esde
machine-learning
MLAir
Commits
b350d93e
You need to sign in or sign up before continuing.
Commit
b350d93e
authored
5 years ago
by
lukas leufen
Browse files
Options
Downloads
Patches
Plain Diff
finalised documentation of both data store classes
parent
22498cea
No related branches found
No related tags found
2 merge requests
!17
update to v0.4.0
,
!14
include data storage
Pipeline
#26368
passed
5 years ago
Stage: test
Stage: pages
Stage: deploy
Changes
2
Pipelines
1
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
.coveragerc
+3
-0
3 additions, 0 deletions
.coveragerc
src/datastore.py
+107
-9
107 additions, 9 deletions
src/datastore.py
with
110 additions
and
9 deletions
.coveragerc
+
3
−
0
View file @
b350d93e
...
@@ -23,3 +23,6 @@ exclude_lines =
...
@@ -23,3 +23,6 @@ exclude_lines =
# Don't complain about import statements
# Don't complain about import statements
import
import
# Don't complain about abstract class declarations and placeholders
pass
This diff is collapsed.
Click to expand it.
src/datastore.py
+
107
−
9
View file @
b350d93e
...
@@ -2,17 +2,21 @@ __author__ = 'Lukas Leufen'
...
@@ -2,17 +2,21 @@ __author__ = 'Lukas Leufen'
__date__
=
'
2019-11-22
'
__date__
=
'
2019-11-22
'
from
typing
import
Any
from
typing
import
Any
,
List
from
abc
import
ABC
from
abc
import
ABC
class
NameNotFoundInDataStore
(
Exception
):
class
NameNotFoundInDataStore
(
Exception
):
"""
Exception that get raised if given name is not found in the entire data store.
"""
pass
pass
class
NameNotFoundInScope
(
Exception
):
class
NameNotFoundInScope
(
Exception
):
"""
Exception that get raised if given name is not found in the provided scope, but can be found in other scopes.
"""
pass
pass
...
@@ -28,21 +32,72 @@ class AbstractDataStore(ABC):
...
@@ -28,21 +32,72 @@ class AbstractDataStore(ABC):
self
.
_store
=
{}
self
.
_store
=
{}
def
put
(
self
,
name
:
str
,
obj
:
Any
,
scope
:
str
)
->
None
:
def
put
(
self
,
name
:
str
,
obj
:
Any
,
scope
:
str
)
->
None
:
"""
Abstract method to add an object to the data store
:param name: Name of object to store
:param obj: The object itself to be stored
:param scope: the scope / context of the object, under that the object is valid
"""
pass
pass
def
get
(
self
,
name
:
str
,
scope
:
str
)
->
None
:
def
get
(
self
,
name
:
str
,
scope
:
str
)
->
None
:
"""
Abstract method to get an object from the data store
:param name: Name to look for
:param scope: scope to search the name for
:return: the stored object
"""
pass
def
search_name
(
self
,
name
:
str
)
->
None
:
"""
Abstract method to search for all occurrences of given `name` in the entire data store.
:param name: Name to look for
:return: search result
"""
pass
pass
class
DataStoreByVariable
(
AbstractDataStore
):
class
DataStoreByVariable
(
AbstractDataStore
):
def
put
(
self
,
name
,
obj
,
scope
):
"""
Data store for all settings for the experiment workflow to save experiment parameters for the proceeding modules
and predefine parameters loaded during the experiment setup phase. The data store is hierarchically structured, so
that global settings can be overwritten by local adjustments.
This implementation stores data as
<variable1>
<scope1>: value
<scope2>: value
<variable2>
<scope1>: value
<scope3>: value
"""
def
put
(
self
,
name
:
str
,
obj
:
Any
,
scope
:
str
)
->
None
:
"""
Store an object `obj` with given `name` under `scope`. In the current implementation, existing entries are
overwritten.
:param name: Name of object to store
:param obj: The object itself to be stored
:param scope: the scope / context of the object, under that the object is valid
"""
# open new variable related store with `name` as key if not existing
# open new variable related store with `name` as key if not existing
if
name
not
in
self
.
_store
.
keys
():
if
name
not
in
self
.
_store
.
keys
():
self
.
_store
[
name
]
=
{}
self
.
_store
[
name
]
=
{}
self
.
_store
[
name
][
scope
]
=
obj
self
.
_store
[
name
][
scope
]
=
obj
def
get
(
self
,
name
,
scope
,
depth
=
None
):
def
get
(
self
,
name
:
str
,
scope
:
str
,
depth
:
int
=
None
)
->
Any
:
"""
Retrieve an object with `name` from `scope`. If no object can be found in the exact scope, take an iterative
look on the levels above. Raises a NameNotFoundInDataStore error, if no object with given name can be found in
the entire data store. Raises a NameNotFoundInScope error, if the object is in the data store but not in the
given scope and its levels above (could be either included in another scope or a more detailed sub-scope).
:param name: Name to look for
:param scope: scope to search the name for
:param depth: counter to check, if all roots of the scope have been visited to trigger an error.
:return: the stored object
"""
if
depth
is
None
:
if
depth
is
None
:
depth
=
scope
.
count
(
"
.
"
)
depth
=
scope
.
count
(
"
.
"
)
if
depth
>=
0
:
if
depth
>=
0
:
...
@@ -57,18 +112,54 @@ class DataStoreByVariable(AbstractDataStore):
...
@@ -57,18 +112,54 @@ class DataStoreByVariable(AbstractDataStore):
else
:
else
:
raise
NameNotFoundInDataStore
(
f
"
Couldn
'
t find
{
name
}
in data store
"
)
raise
NameNotFoundInDataStore
(
f
"
Couldn
'
t find
{
name
}
in data store
"
)
def
search_name
(
self
,
name
):
def
search_name
(
self
,
name
:
str
)
->
List
[
str
]:
"""
Search for all occurrences of given `name` in the entire data store.
:param name: Name to look for
:return: list with all scopes and sub-scopes containing an object stored as `name`
"""
return
sorted
(
self
.
_store
[
name
])
return
sorted
(
self
.
_store
[
name
])
class
DataStoreByScope
(
AbstractDataStore
):
class
DataStoreByScope
(
AbstractDataStore
):
def
put
(
self
,
name
,
obj
,
scope
):
"""
Data store for all settings for the experiment workflow to save experiment parameters for the proceeding modules
and predefine parameters loaded during the experiment setup phase. The data store is hierarchically structured, so
that global settings can be overwritten by local adjustments.
This implementation stores data as
<scope1>
<variable1>: value
<variable2>: value
<scope2>
<variable1>: value
<variable3>: value
"""
def
put
(
self
,
name
:
str
,
obj
:
Any
,
scope
:
str
)
->
None
:
"""
Store an object `obj` with given `name` under `scope`. In the current implementation, existing entries are
overwritten.
:param name: Name of object to store
:param obj: The object itself to be stored
:param scope: the scope / context of the object, under that the object is valid
"""
if
scope
not
in
self
.
_store
.
keys
():
if
scope
not
in
self
.
_store
.
keys
():
self
.
_store
[
scope
]
=
{}
self
.
_store
[
scope
]
=
{}
self
.
_store
[
scope
][
name
]
=
obj
self
.
_store
[
scope
][
name
]
=
obj
def
get
(
self
,
name
,
scope
,
depth
=
None
):
def
get
(
self
,
name
:
str
,
scope
:
str
,
depth
:
int
=
None
)
->
Any
:
"""
Retrieve an object with `name` from `scope`. If no object can be found in the exact scope, take an iterative
look on the levels above. Raises a NameNotFoundInDataStore error, if no object with given name can be found in
the entire data store. Raises a NameNotFoundInScope error, if the object is in the data store but not in the
given scope and its levels above (could be either included in another scope or a more detailed sub-scope).
:param name: Name to look for
:param scope: scope to search the name for
:param depth: counter to check, if all roots of the scope have been visited to trigger an error.
:return: the stored object
"""
if
depth
is
None
:
if
depth
is
None
:
depth
=
scope
.
count
(
"
.
"
)
depth
=
scope
.
count
(
"
.
"
)
if
depth
>=
0
:
if
depth
>=
0
:
...
@@ -84,9 +175,16 @@ class DataStoreByScope(AbstractDataStore):
...
@@ -84,9 +175,16 @@ class DataStoreByScope(AbstractDataStore):
raise
NameNotFoundInScope
(
f
"
Couldn
'
t find
{
name
}
in scope
{
scope
}
.
{
name
}
is only defined in
"
raise
NameNotFoundInScope
(
f
"
Couldn
'
t find
{
name
}
in scope
{
scope
}
.
{
name
}
is only defined in
"
f
"
{
occurrences
}
"
)
f
"
{
occurrences
}
"
)
def
search_name
(
self
,
name
):
def
search_name
(
self
,
name
:
str
)
->
List
[
str
]:
"""
Search for all occurrences of given `name` in the entire data store.
:param name: Name to look for
:return: list with all scopes and sub-scopes containing an object stored as `name`
"""
keys
=
[]
keys
=
[]
for
(
key
,
val
)
in
self
.
_store
.
items
():
for
(
key
,
val
)
in
self
.
_store
.
items
():
if
name
in
val
.
keys
():
if
name
in
val
.
keys
():
keys
.
append
(
key
)
keys
.
append
(
key
)
return
sorted
(
keys
)
return
sorted
(
keys
)
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment