Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
B
b2share_depositer
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Martin Schultz
b2share_depositer
Commits
fc629a60
Commit
fc629a60
authored
Dec 19, 2016
by
Benedikt von St. Vieth
Browse files
Options
Downloads
Patches
Plain Diff
enhanced upload, output, renaming of deposited files now possible.
parent
9d9fa52a
Branches
Branches containing commit
No related tags found
No related merge requests found
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
README.md
+24
-2
24 additions, 2 deletions
README.md
depositor/b2share.py
+57
-21
57 additions, 21 deletions
depositor/b2share.py
with
81 additions
and
23 deletions
README.md
+
24
−
2
View file @
fc629a60
sample
python script for automatically depositing files to B2SHARE v2
python script for
(more or less)
automatically depositing files to B2SHARE v2
still work in progress
######
python run.py --apitoken $TOKEN --filenames
<file_a.txt
,
file_b.txt
>
--b2shareurl (change the current default b2share url fsd-cloud9.zam.kfa-juelich.de)
--community $id (e.g. e9b9792e-79fb-4b07-b6b4-b9c2bd06d095)
--debug (to get some debug output. ATTENTION: tokens are shown due to python requests)
--filenames
A comma separated list of files to upload. URLs are also supported, at least
for openstack swift streaming worked well.
If you want to rename a file showing up in the deposit, write
filename_a::newfilename_a (use two double points)
--finalize (whether to finalize the deposit, not always possible without additional metadata)
--insecure (whether to use a insecure API endpoint)
# streaming from http resource to b2share should work for http/https resources
python run.py --apitoken $TOKEN --filenames https://swift.zam.kfa-juelich.de:8889/v1/AUTH_acf10a23f9af48ec961c5354888cc212/public/unicore-servers-7.4.0.tgz
This diff is collapsed.
Click to expand it.
depositor/b2share.py
+
57
−
21
View file @
fc629a60
"""
a b2share depositor.
"""
"""
a b2share depositor.
"""
from
furl
import
furl
from
furl
import
furl
import
json
import
json
import
logging
import
logging
import
pprint
import
requests
import
requests
import
sys
import
sys
...
@@ -35,18 +35,32 @@ class Depositor:
...
@@ -35,18 +35,32 @@ class Depositor:
self
.
community
=
self
.
select_community
(
communitynames
,
self
.
community
=
self
.
select_community
(
communitynames
,
communityids
)
communityids
)
deposit_id
,
upload_url
=
self
.
create_deposit
()
deposit_id
,
upload_url
=
self
.
create_deposit
()
logging
.
debug
(
"
Deposit has id {} and files should be
"
"
uploaded to {}
"
.
format
(
deposit_id
,
upload_url
))
self
.
upload_files
(
self
.
filenames
,
upload_url
)
self
.
upload_files
(
self
.
filenames
,
upload_url
)
finalized
=
self
.
finalize_deposit
()
if
finalized
:
output
=
'
Deposit available at: {}{}
'
.
format
(
self
.
b2shareurl
.
replace
(
'
api/
'
,
'
records/
'
),
deposit_id
)
else
:
output
=
'
Deposit available to finalize at: {}{}/edit
'
\
.
format
(
self
.
b2shareurl
.
replace
(
'
api/
'
,
'
records/
'
),
deposit_id
)
logging
.
info
(
'
{}
'
.
format
(
output
))
def
create_deposit
(
self
):
def
create_deposit
(
self
):
"""
create initial deposit based on provided or selected community.
"""
"""
create initial deposit based on provided or selected community.
"""
headers
=
{
'
Content-Type
'
:
'
application/json
'
}
headers
=
{
'
Content-Type
'
:
'
application/json
'
}
metadata
=
{
'
community
'
:
self
.
community
}
metadata
=
{
'
community
'
:
self
.
community
}
while
True
:
while
True
:
try
:
title
=
user_input
(
"
Please enter the title of your deposit:
"
)
title
=
user_input
(
"
Please enter the title of your deposit:
"
)
if
title
:
if
title
:
break
break
except
EOFError
:
logging
.
info
(
"
Exiting...
"
)
exit
(
0
)
except
KeyboardInterrupt
:
logging
.
info
(
"
Exiting...
"
)
exit
(
0
)
metadata
.
update
({
'
title
'
:
title
.
strip
()})
metadata
.
update
({
'
title
'
:
title
.
strip
()})
while
True
:
while
True
:
input_open_access
=
user_input
(
"
Grant open access? [Y/n]
"
).
strip
()
input_open_access
=
user_input
(
"
Grant open access? [Y/n]
"
).
strip
()
...
@@ -63,9 +77,9 @@ class Depositor:
...
@@ -63,9 +77,9 @@ class Depositor:
r
=
requests
.
post
(
self
.
b2shareurl
+
"
records/
"
,
r
=
requests
.
post
(
self
.
b2shareurl
+
"
records/
"
,
params
=
self
.
url_params
,
data
=
json
.
dumps
(
metadata
),
params
=
self
.
url_params
,
data
=
json
.
dumps
(
metadata
),
headers
=
headers
,
verify
=
self
.
insecure_ssl
)
headers
=
headers
,
verify
=
self
.
insecure_ssl
)
self
.
deposit
=
r
.
json
()[
'
id
'
]
deposit
_id
=
r
.
json
()[
'
id
'
]
self
.
upload_url
=
r
.
json
()[
'
links
'
][
'
files
'
]
upload_url
=
r
.
json
()[
'
links
'
][
'
files
'
]
return
self
.
deposit
,
self
.
upload_url
return
deposit
_id
,
upload_url
def
fetch_community_list
(
self
):
def
fetch_community_list
(
self
):
"""
fetch a list of current communities from b2share.
"""
"""
fetch a list of current communities from b2share.
"""
...
@@ -82,7 +96,8 @@ class Depositor:
...
@@ -82,7 +96,8 @@ class Depositor:
i
+=
1
i
+=
1
return
communitynames
,
communityids
return
communitynames
,
communityids
def
select_community
(
self
,
communitynames
,
communityids
):
@classmethod
def
select_community
(
cls
,
communitynames
,
communityids
):
"""
select a community from current b2share communities.
"""
"""
select a community from current b2share communities.
"""
for
i
in
range
(
0
,
len
(
communitynames
)):
for
i
in
range
(
0
,
len
(
communitynames
)):
print
(
'
{int:2d}:
'
.
format
(
int
=
i
)
+
communitynames
[
i
])
print
(
'
{int:2d}:
'
.
format
(
int
=
i
)
+
communitynames
[
i
])
...
@@ -95,6 +110,9 @@ class Depositor:
...
@@ -95,6 +110,9 @@ class Depositor:
except
EOFError
:
except
EOFError
:
logging
.
info
(
"
Exiting...
"
)
logging
.
info
(
"
Exiting...
"
)
exit
(
0
)
exit
(
0
)
except
KeyboardInterrupt
:
logging
.
info
(
"
Exiting...
"
)
exit
(
0
)
else
:
else
:
if
selected_community
<
0
or
selected_community
>=
\
if
selected_community
<
0
or
selected_community
>=
\
len
(
communitynames
):
len
(
communitynames
):
...
@@ -105,22 +123,40 @@ class Depositor:
...
@@ -105,22 +123,40 @@ class Depositor:
break
break
return
communityids
[
selected_community
]
return
communityids
[
selected_community
]
def
finalize_deposit
(
self
):
@classmethod
def
finalize_deposit
(
cls
):
"""
close (publish) the deposit.
"""
"""
close (publish) the deposit.
"""
pass
return
False
def
upload_files
(
self
,
filenames
,
upload_url
):
def
upload_files
(
self
,
filenames
,
upload_url
):
"""
upload files, start with one file.
"""
"""
upload files, start with one file.
"""
headers
=
{
'
Content-Type
'
:
'
application/octet-stream
'
}
headers
=
{
'
Content-Type
'
:
'
application/octet-stream
'
}
for
filename
in
filenames
:
for
filename
in
filenames
:
if
'
::
'
in
filename
:
logging
.
info
(
filename
.
count
(
'
::
'
))
if
not
filename
.
count
(
'
::
'
)
is
1
:
logging
.
error
(
'
Too many
"
::
"
for filename renaming of {}
'
'
given, skipping.
'
.
format
(
filename
))
continue
source
,
target
=
filename
.
split
(
'
::
'
)
else
:
source
=
target
=
filename
if
source
.
startswith
(
'
https
'
)
or
source
.
startswith
(
'
http
'
):
logging
.
debug
(
'
Streaming http/s object
'
)
r
=
requests
.
get
(
source
,
stream
=
True
)
requests
.
put
(
upload_url
+
'
/
'
+
target
,
params
=
self
.
url_params
,
data
=
r
,
headers
=
headers
,
verify
=
self
.
insecure_ssl
)
else
:
try
:
try
:
with
open
(
filename
,
'
rb
'
)
as
f
:
with
open
(
filename
,
'
rb
'
)
as
f
:
requests
.
put
(
upload_url
+
'
/
'
+
filename
,
requests
.
put
(
upload_url
+
'
/
'
+
target
,
params
=
self
.
url_params
,
data
=
f
,
params
=
self
.
url_params
,
data
=
f
,
headers
=
headers
,
verify
=
self
.
insecure_ssl
)
headers
=
headers
,
verify
=
self
.
insecure_ssl
)
except
IOError
:
except
IOError
:
logging
.
error
(
"
File with name
'
{}
'
does not exist,
"
logging
.
error
(
"
File with name
'
{}
'
does not exist,
"
"
skipping
"
.
format
(
filenam
e
))
"
skipping
"
.
format
(
sourc
e
))
def
sanitize_api_url
(
url
):
def
sanitize_api_url
(
url
):
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment