Skip to content
Snippets Groups Projects
Commit fc629a60 authored by Benedikt von St. Vieth's avatar Benedikt von St. Vieth
Browse files

enhanced upload, output, renaming of deposited files now possible.

parent 9d9fa52a
Branches
No related tags found
No related merge requests found
sample python script for automatically depositing files to B2SHARE v2 python script for (more or less) automatically depositing files to B2SHARE v2
still work in progress ######
python run.py --apitoken $TOKEN --filenames <file_a.txt,file_b.txt>
--b2shareurl (change the current default b2share url fsd-cloud9.zam.kfa-juelich.de)
--community $id (e.g. e9b9792e-79fb-4b07-b6b4-b9c2bd06d095)
--debug (to get some debug output. ATTENTION: tokens are shown due to python requests)
--filenames
A comma separated list of files to upload. URLs are also supported, at least
for openstack swift streaming worked well.
If you want to rename a file showing up in the deposit, write
filename_a::newfilename_a (use two double points)
--finalize (whether to finalize the deposit, not always possible without additional metadata)
--insecure (whether to use a insecure API endpoint)
# streaming from http resource to b2share should work for http/https resources
python run.py --apitoken $TOKEN --filenames https://swift.zam.kfa-juelich.de:8889/v1/AUTH_acf10a23f9af48ec961c5354888cc212/public/unicore-servers-7.4.0.tgz
"""a b2share depositor.""" """a b2share depositor."""
from furl import furl from furl import furl
import json import json
import logging import logging
import pprint
import requests import requests
import sys import sys
...@@ -35,18 +35,32 @@ class Depositor: ...@@ -35,18 +35,32 @@ class Depositor:
self.community = self.select_community(communitynames, self.community = self.select_community(communitynames,
communityids) communityids)
deposit_id, upload_url = self.create_deposit() deposit_id, upload_url = self.create_deposit()
logging.debug("Deposit has id {} and files should be "
"uploaded to {}".format(deposit_id, upload_url))
self.upload_files(self.filenames, upload_url) self.upload_files(self.filenames, upload_url)
finalized = self.finalize_deposit()
if finalized:
output = 'Deposit available at: {}{}'.format(
self.b2shareurl.replace('api/', 'records/'), deposit_id)
else:
output = 'Deposit available to finalize at: {}{}/edit' \
.format(self.b2shareurl.replace('api/', 'records/'), deposit_id)
logging.info('{}'.format(output))
def create_deposit(self): def create_deposit(self):
"""create initial deposit based on provided or selected community.""" """create initial deposit based on provided or selected community."""
headers = {'Content-Type': 'application/json'} headers = {'Content-Type': 'application/json'}
metadata = {'community': self.community} metadata = {'community': self.community}
while True: while True:
try:
title = user_input("Please enter the title of your deposit: ") title = user_input("Please enter the title of your deposit: ")
if title: if title:
break break
except EOFError:
logging.info("Exiting...")
exit(0)
except KeyboardInterrupt:
logging.info("Exiting...")
exit(0)
metadata.update({'title': title.strip()}) metadata.update({'title': title.strip()})
while True: while True:
input_open_access = user_input("Grant open access? [Y/n] ").strip() input_open_access = user_input("Grant open access? [Y/n] ").strip()
...@@ -63,9 +77,9 @@ class Depositor: ...@@ -63,9 +77,9 @@ class Depositor:
r = requests.post(self.b2shareurl + "records/", r = requests.post(self.b2shareurl + "records/",
params=self.url_params, data=json.dumps(metadata), params=self.url_params, data=json.dumps(metadata),
headers=headers, verify=self.insecure_ssl) headers=headers, verify=self.insecure_ssl)
self.deposit = r.json()['id'] deposit_id = r.json()['id']
self.upload_url = r.json()['links']['files'] upload_url = r.json()['links']['files']
return self.deposit, self.upload_url return deposit_id, upload_url
def fetch_community_list(self): def fetch_community_list(self):
"""fetch a list of current communities from b2share.""" """fetch a list of current communities from b2share."""
...@@ -82,7 +96,8 @@ class Depositor: ...@@ -82,7 +96,8 @@ class Depositor:
i += 1 i += 1
return communitynames, communityids return communitynames, communityids
def select_community(self, communitynames, communityids): @classmethod
def select_community(cls, communitynames, communityids):
"""select a community from current b2share communities.""" """select a community from current b2share communities."""
for i in range(0, len(communitynames)): for i in range(0, len(communitynames)):
print('{int:2d}: '.format(int=i) + communitynames[i]) print('{int:2d}: '.format(int=i) + communitynames[i])
...@@ -95,6 +110,9 @@ class Depositor: ...@@ -95,6 +110,9 @@ class Depositor:
except EOFError: except EOFError:
logging.info("Exiting...") logging.info("Exiting...")
exit(0) exit(0)
except KeyboardInterrupt:
logging.info("Exiting...")
exit(0)
else: else:
if selected_community < 0 or selected_community >= \ if selected_community < 0 or selected_community >= \
len(communitynames): len(communitynames):
...@@ -105,22 +123,40 @@ class Depositor: ...@@ -105,22 +123,40 @@ class Depositor:
break break
return communityids[selected_community] return communityids[selected_community]
def finalize_deposit(self): @classmethod
def finalize_deposit(cls):
"""close (publish) the deposit.""" """close (publish) the deposit."""
pass return False
def upload_files(self, filenames, upload_url): def upload_files(self, filenames, upload_url):
"""upload files, start with one file.""" """upload files, start with one file."""
headers = {'Content-Type': 'application/octet-stream'} headers = {'Content-Type': 'application/octet-stream'}
for filename in filenames: for filename in filenames:
if '::' in filename:
logging.info(filename.count('::'))
if not filename.count('::') is 1:
logging.error('Too many "::" for filename renaming of {} '
'given, skipping.'.format(filename))
continue
source, target = filename.split('::')
else:
source = target = filename
if source.startswith('https') or source.startswith('http'):
logging.debug('Streaming http/s object')
r = requests.get(source, stream=True)
requests.put(upload_url + '/' + target,
params=self.url_params, data=r,
headers=headers, verify=self.insecure_ssl)
else:
try: try:
with open(filename, 'rb') as f: with open(filename, 'rb') as f:
requests.put(upload_url + '/' + filename, requests.put(upload_url + '/' + target,
params=self.url_params, data=f, params=self.url_params, data=f,
headers=headers, verify=self.insecure_ssl) headers=headers, verify=self.insecure_ssl)
except IOError: except IOError:
logging.error("File with name '{}' does not exist, " logging.error("File with name '{}' does not exist, "
"skipping".format(filename)) "skipping".format(source))
def sanitize_api_url(url): def sanitize_api_url(url):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment