Skip to content
Snippets Groups Projects
Commit d582f3a3 authored by Jedrzej Rybicki's avatar Jedrzej Rybicki
Browse files

b2share operator using more airflowish hook

parent 16254e9f
Branches
No related tags found
No related merge requests found
from airflow.models.baseoperator import BaseOperator from airflow.models.baseoperator import BaseOperator
from airflow.models.connection import Connection from airflow.models.connection import Connection
from airflow.providers.http.hooks.http import HttpHook
import requests import requests
from urllib.parse import urljoin from urllib.parse import urljoin
import tempfile import tempfile
...@@ -42,19 +43,15 @@ class B2ShareOperator(BaseOperator): ...@@ -42,19 +43,15 @@ class B2ShareOperator(BaseOperator):
print(self.target_dir) print(self.target_dir)
def execute(self, **kwargs): def execute(self, **kwargs):
connection = Connection.get_connection_from_secrets('default_b2share') hook = HttpHook(http_conn_id=self.conn_id, method='GET')
server = connection.get_uri()
print(f"Rereiving data from {server}")
print('Kwargs')
print(kwargs)
params = kwargs['context']['params'] params = kwargs['context']['params']
oid = params['oid'] oid = params['oid']
obj = get_object_md(server=server, oid=oid) hrespo = hook.run(endpoint=f"/api/records/{oid}")
print(f"Retrieved object {oid}: {obj}") print(hrespo)
flist = get_file_list(obj)
flist = get_file_list(hrespo.json())
print(flist)
ti = kwargs['context']['ti'] ti = kwargs['context']['ti']
name_mappings = {} name_mappings = {}
for fname, url in flist.items(): for fname, url in flist.items():
...@@ -66,5 +63,5 @@ class B2ShareOperator(BaseOperator): ...@@ -66,5 +63,5 @@ class B2ShareOperator(BaseOperator):
ti.xcom_push(key='remote', value=fname) ti.xcom_push(key='remote', value=fname)
break # for now only one file break # for now only one file
ti.xcom_push(key='mappins', value=name_mappings)
return len(name_mappings) return len(name_mappings)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment