From 03cb6f4ed35a7fb3534ec89630fba17d3fc5d58f Mon Sep 17 00:00:00 2001
From: jrybicki-jsc <j.rybicki@fz-juelich.de>
Date: Wed, 6 Apr 2022 09:02:54 +0200
Subject: [PATCH] working with datacat hook
---
dags/just_reg.py | 2 +-
dags/uploadflow.py | 22 ++++++++++++++--------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/dags/just_reg.py b/dags/just_reg.py
index 877e386..827eef9 100644
--- a/dags/just_reg.py
+++ b/dags/just_reg.py
@@ -80,7 +80,7 @@ def datacat_registration_example():
print(entry)
print(entry['metadata'])
print('---')
- print(get_template(entry['metadata']))
+ print(create_template(entry['metadata']))
diff --git a/dags/uploadflow.py b/dags/uploadflow.py
index 18c5043..4ceacb9 100644
--- a/dags/uploadflow.py
+++ b/dags/uploadflow.py
@@ -1,7 +1,7 @@
import os
import tempfile
-from urllib.parse import urljoin
+import json
from airflow.decorators import dag, task
from airflow.models import Variable
@@ -78,12 +78,18 @@ def upload_example():
params = kwargs['params']
mid = params['mid']
- hook = HttpHook(http_conn_id='datacat', method='GET')
- hrespo = hook.run(endpoint=f"storage_target/{mid}").json()['metadata']
- print(hrespo)
- template = create_template(hrespo=hrespo)
- community = get_community(
- server=server, community_id=template['community'])
+
+ hook = DataCatalogHook()
+ print("Connected to datacat via hook", hook.list_type('dataset'))
+ entry = json.loads(hook.get_entry(datacat_type='storage_target', oid=mid))
+
+ #hook = HttpHook(http_conn_id='datacat', method='GET')
+ #hrespo = hook.run(endpoint=f"storage_target/{mid}").json()['metadata']
+ print('Got following metadata', entry)
+
+ template = create_template(hrespo=entry['metadata'])
+ community = get_community(server=server,
+ community_id=template['community'])
if not community:
print("Not existing community")
return -1
@@ -129,7 +135,7 @@ def upload_example():
)
try:
r = hook.create_entry(datacat_type='dataset', entry=entry)
- print("Hook registration returned: ", r, urljoin(hook.connection.url, r))
+ print("Hook registration returned: ", r)
return r
except ConnectionError as e:
print('Registration failed', e)
--
GitLab