Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
D
DataCatalog
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
eFlows4HPC WP2
DataCatalog
Commits
4d039833
Commit
4d039833
authored
3 years ago
by
Jedrzej Rybicki
Browse files
Options
Downloads
Patches
Plain Diff
all pydantinc now. looks better
parent
b1e18ee1
Branches
Branches containing commit
Tags
Tags containing commit
No related merge requests found
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
apiserver/storage/JsonFileStorageAdapter.py
+48
-49
48 additions, 49 deletions
apiserver/storage/JsonFileStorageAdapter.py
tests/storage_tests/test_jsonbackend.py
+23
-5
23 additions, 5 deletions
tests/storage_tests/test_jsonbackend.py
with
71 additions
and
54 deletions
apiserver/storage/JsonFileStorageAdapter.py
+
48
−
49
View file @
4d039833
...
@@ -8,15 +8,17 @@ from apiserver.config import ApiserverSettings
...
@@ -8,15 +8,17 @@ from apiserver.config import ApiserverSettings
from
.LocationStorage
import
(
AbstractLocationDataStorageAdapter
,
LocationData
,
from
.LocationStorage
import
(
AbstractLocationDataStorageAdapter
,
LocationData
,
LocationDataType
)
LocationDataType
)
#from dataclasses import dataclass, asdict
#perhaps even that:
#from dataclasses_json import dataclass_json
class
StoredData
:
from
pydantic
import
BaseModel
#@dataclass
class
StoredData
(
BaseModel
):
actualData
:
LocationData
actualData
:
LocationData
users
:
List
[
str
]
users
:
List
[
str
]
def
toDict
(
self
):
return
{
'
actualData
'
:
self
.
actualData
.
__dict__
,
'
users
'
:
self
.
users
}
class
JsonFileStorageAdapter
(
AbstractLocationDataStorageAdapter
):
class
JsonFileStorageAdapter
(
AbstractLocationDataStorageAdapter
):
"""
This stores LocationData via the StoredData Object as json files
"""
This stores LocationData via the StoredData Object as json files
...
@@ -35,14 +37,24 @@ class JsonFileStorageAdapter(AbstractLocationDataStorageAdapter):
...
@@ -35,14 +37,24 @@ class JsonFileStorageAdapter(AbstractLocationDataStorageAdapter):
if
not
(
os
.
path
.
exists
(
self
.
data_dir
)
and
os
.
path
.
isdir
(
self
.
data_dir
)):
if
not
(
os
.
path
.
exists
(
self
.
data_dir
)
and
os
.
path
.
isdir
(
self
.
data_dir
)):
raise
Exception
(
f
"
Data Directory
{
self
.
data_dir
}
does not exist.
"
)
raise
Exception
(
f
"
Data Directory
{
self
.
data_dir
}
does not exist.
"
)
def
__setup_path
(
self
,
value
)
:
def
__setup_path
(
self
,
value
:
str
)
->
str
:
localpath
=
os
.
path
.
join
(
self
.
data_dir
,
value
)
localpath
=
os
.
path
.
join
(
self
.
data_dir
,
value
)
if
not
(
os
.
path
.
isdir
(
localpath
)):
if
not
(
os
.
path
.
isdir
(
localpath
)):
# This type has apparently not yet been used at all,
# create its directory and return an empty json file
os
.
mkdir
(
localpath
)
os
.
mkdir
(
localpath
)
return
localpath
return
localpath
def
__get_object_path
(
self
,
value
:
str
,
oid
:
str
)
->
str
:
localpath
=
os
.
path
.
join
(
self
.
data_dir
,
value
)
fullpath
=
os
.
path
.
join
(
localpath
,
oid
)
if
not
os
.
path
.
isfile
(
fullpath
):
raise
FileNotFoundError
(
f
"
The requested object (
{
oid
}
) does not exist.
"
)
return
fullpath
def
__get_unique_id
(
self
,
path
:
str
)
->
str
:
oid
=
str
(
uuid
.
uuid4
())
while
(
os
.
path
.
exists
(
os
.
path
.
join
(
path
,
oid
))):
oid
=
str
(
uuid
.
uuid4
())
return
oid
def
get_list
(
self
,
n_type
:
LocationDataType
)
->
List
:
def
get_list
(
self
,
n_type
:
LocationDataType
)
->
List
:
local_path
=
self
.
__setup_path
(
n_type
.
value
)
local_path
=
self
.
__setup_path
(
n_type
.
value
)
...
@@ -52,59 +64,46 @@ class JsonFileStorageAdapter(AbstractLocationDataStorageAdapter):
...
@@ -52,59 +64,46 @@ class JsonFileStorageAdapter(AbstractLocationDataStorageAdapter):
# and the LocationData name (which is inside the json)
# and the LocationData name (which is inside the json)
retList
=
[]
retList
=
[]
for
f
in
allFiles
:
for
f
in
allFiles
:
with
open
(
os
.
path
.
join
(
local_path
,
f
))
as
file
:
data
=
StoredData
.
parse_file
(
os
.
path
.
join
(
local_path
,
f
))
data
=
json
.
load
(
file
)
#with open(os.path.join(local_path, f)) as file:
retList
.
append
({
data
[
'
actualData
'
][
'
name
'
]:
f
})
# data = json.load(file)
retList
.
append
({
data
.
actualData
.
name
:
f
})
return
retList
return
retList
def
add_new
(
self
,
n_type
:
LocationDataType
,
data
:
LocationData
,
usr
:
str
):
def
add_new
(
self
,
n_type
:
LocationDataType
,
data
:
LocationData
,
usr
:
str
):
localpath
=
self
.
__setup_path
(
value
=
n_type
.
value
)
localpath
=
self
.
__setup_path
(
value
=
n_type
.
value
)
# create a unique oid, by randomly generating one,
oid
=
self
.
__get_unique_id
(
path
=
localpath
)
# and re-choosing if it is already taken
toStore
=
StoredData
(
users
=
[
usr
],
actualData
=
data
)
oid
=
str
(
uuid
.
uuid4
())
while
(
os
.
path
.
exists
(
os
.
path
.
join
(
localpath
,
oid
))):
oid
=
str
(
uuid
.
uuid4
())
toStore
=
StoredData
()
toStore
.
users
=
[
usr
]
toStore
.
actualData
=
data
with
open
(
os
.
path
.
join
(
localpath
,
oid
),
'
w
'
)
as
json_file
:
with
open
(
os
.
path
.
join
(
localpath
,
oid
),
'
w
'
)
as
json_file
:
json
.
dump
(
toStore
.
toD
ict
(),
json_file
)
json
.
dump
(
toStore
.
d
ict
(),
json_file
)
return
{
oid
:
data
}
return
(
oid
,
data
)
def
get_details
(
self
,
type
:
LocationDataType
,
oid
:
str
):
def
__load_object
(
self
,
path
):
localpath
=
os
.
path
.
join
(
self
.
data_dir
,
type
.
value
)
# move to data class?
fullpath
=
os
.
path
.
join
(
localpath
,
oid
)
with
open
(
path
,
'
r
'
)
as
f
:
if
not
os
.
path
.
isfile
(
fullpath
):
data
=
json
.
load
(
f
)
raise
FileNotFoundError
(
f
"
The requested object (
{
oid
}
) does not exist.
"
)
return
data
with
open
(
fullpath
)
as
file
:
data
=
json
.
load
(
file
)
return
data
[
'
actualData
'
]
def
update_details
(
self
,
type
:
LocationDataType
,
oid
:
str
,
data
:
LocationData
,
usr
:
str
):
localpath
=
os
.
path
.
join
(
self
.
data_dir
,
type
.
value
)
fullpath
=
os
.
path
.
join
(
localpath
,
oid
)
if
not
os
.
path
.
isfile
(
fullpath
):
raise
FileNotFoundError
(
f
"
The requested object (
{
oid
}
) does not exist.
"
)
toStore
=
StoredData
()
def
get_details
(
self
,
n_type
:
LocationDataType
,
oid
:
str
):
toStore
.
actualData
=
data
full_path
=
self
.
__get_object_path
(
value
=
n_type
.
value
,
oid
=
oid
)
obj
=
self
.
__load_object
(
path
=
full_path
)
return
obj
[
'
actualData
'
]
# get permissions from old file
with
open
(
fullpath
)
as
file
:
old_data
=
json
.
load
(
file
)
toStore
.
users
=
old_data
[
'
users
'
]
with
open
(
fullpath
,
'
w
'
)
as
file
:
def
update_details
(
self
,
n_type
:
LocationDataType
,
oid
:
str
,
data
:
LocationData
,
usr
:
str
):
json
.
dump
(
toStore
.
toDict
(),
file
)
full_path
=
self
.
__get_object_path
(
value
=
n_type
.
value
,
oid
=
oid
)
return
{
oid
:
data
}
old_data
=
self
.
__load_object
(
path
=
full_path
)
old_data
[
'
actualData
'
]
=
data
def
delete
(
self
,
type
:
LocationDataType
,
oid
:
str
,
usr
:
str
):
full_path
=
self
.
__get_object_path
(
value
=
n_type
.
value
,
oid
=
oid
)
localpath
=
os
.
path
.
join
(
self
.
data_dir
,
type
.
value
)
with
open
(
full_path
,
'
w
'
)
as
file
:
fullpath
=
os
.
p
at
h
.
j
oin
(
localpath
,
oid
)
json
.
dump
(
old_d
at
a
.
j
son
(),
file
)
if
not
os
.
path
.
isfile
(
fullpath
):
return
(
oid
,
data
)
raise
FileNotFoundError
(
f
"
The requested object
{
oid
}
does not exist.
"
)
def
delete
(
self
,
n_type
:
LocationDataType
,
oid
:
str
,
usr
:
str
):
fullpath
=
self
.
__get_object_path
(
value
=
n_type
.
value
,
oid
=
oid
)
os
.
remove
(
fullpath
)
os
.
remove
(
fullpath
)
def
get_owner
(
self
,
type
:
LocationDataType
,
oid
:
str
):
def
get_owner
(
self
,
type
:
LocationDataType
,
oid
:
str
):
...
...
This diff is collapsed.
Click to expand it.
tests/storage_tests/test_jsonbackend.py
+
23
−
5
View file @
4d039833
import
unittest
import
unittest
from
apiserver.storage.JsonFileStorageAdapter
import
JsonFileStorageAdapter
from
apiserver.storage.JsonFileStorageAdapter
import
JsonFileStorageAdapter
,
StoredData
from
apiserver.storage
import
LocationDataType
from
apiserver.storage
import
LocationDataType
,
LocationData
from
collections
import
namedtuple
from
collections
import
namedtuple
import
os
import
os
import
pathlib
import
pathlib
...
@@ -31,6 +31,24 @@ class SomeTests(unittest.TestCase):
...
@@ -31,6 +31,24 @@ class SomeTests(unittest.TestCase):
test_config
=
Settings
(
'
/tmp/json_test/blah/
'
)
test_config
=
Settings
(
'
/tmp/json_test/blah/
'
)
self
.
assertRaises
(
Exception
,
JsonFileStorageAdapter
,
test_config
)
self
.
assertRaises
(
Exception
,
JsonFileStorageAdapter
,
test_config
)
def
test_add_new
(
self
):
d
=
LocationData
(
name
=
'
bla
'
,
url
=
'
local
'
)
(
oid
,
data
)
=
self
.
store
.
add_new
(
n_type
=
LocationDataType
.
DATASET
,
data
=
d
,
usr
=
'
test_user
'
)
self
.
assertEquals
(
d
,
data
,
"
Data should be equal
"
)
self
.
assertIsNotNone
(
oid
)
def
test_add_and_read
(
self
):
l_data
=
LocationData
(
name
=
'
test1
'
,
url
=
'
http://n.go
'
,
metadata
=
[])
(
oid
,
data
)
=
self
.
store
.
add_new
(
n_type
=
LocationDataType
.
DATASET
,
data
=
l_data
,
usr
=
'
test_user
'
)
self
.
assertEquals
(
l_data
,
data
,
"
Data should be equal
"
)
self
.
assertIsNotNone
(
oid
)
print
(
data
)
lst
=
self
.
store
.
get_list
(
n_type
=
LocationDataType
.
DATASET
)
self
.
assertEqual
(
len
(
lst
),
1
,
'
One should be there
'
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment